Merge pull request #107 from lbryio/metadata-version

Metadata versioning, publishing things, and USD denominated key fees

lbrynet/__init__.py

@@ -2,7 +2,7 @@ import logging
 
 log = logging.getLogger(__name__)
 logging.getLogger(__name__).addHandler(logging.NullHandler())
-log.setLevel(logging.ERROR)
+log.setLevel(logging.INFO)
 
 __version__ = "0.3.11"
 version = tuple(__version__.split('.'))

lbrynet/conf.py

@@ -10,6 +10,7 @@ MAX_RESPONSE_INFO_SIZE = 2**16
 MAX_BLOB_INFOS_TO_REQUEST = 20
 BLOBFILES_DIR = ".blobfiles"
 BLOB_SIZE = 2**21
+
 MIN_BLOB_DATA_PAYMENT_RATE = .005  # points/megabyte
 MIN_BLOB_INFO_PAYMENT_RATE = .02  # points/1000 infos
 MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05  # points/1000 infos
@@ -23,6 +24,9 @@ KNOWN_DHT_NODES = [('104.236.42.182', 4000),
 
 POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
 #POINTTRADER_SERVER = 'http://127.0.0.1:2424'
+SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005",
+                  "http://lighthouse2.lbry.io:50005",
+                  "http://lighthouse3.lbry.io:50005"]
 
 LOG_FILE_NAME = "lbrynet.log"
 LOG_POST_URL = "https://lbry.io/log-upload"
@@ -42,11 +46,14 @@ DEFAULT_WALLET = "lbryum"
 WALLET_TYPES = ["lbryum", "lbrycrd"]
 DEFAULT_TIMEOUT = 30
 DEFAULT_MAX_SEARCH_RESULTS = 25
-DEFAULT_MAX_KEY_FEE = 100.0
+DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}}
 DEFAULT_SEARCH_TIMEOUT = 3.0
 DEFAULT_CACHE_TIME = 3600
 DEFAULT_UI_BRANCH = "master"
 
 SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
-BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type']
+CURRENCIES = {
-OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey']
+                'BTC': {'type': 'crypto'},
+                'LBC': {'type': 'crypto'},
+                'USD': {'type': 'fiat'},
+             }

lbrynet/core/Error.py

@@ -22,6 +22,10 @@ class ConnectionClosedBeforeResponseError(Exception):
     pass
 
 
+class KeyFeeAboveMaxAllowed(Exception):
+    pass
+
+
 class UnknownNameError(Exception):
     def __init__(self, name):
         self.name = name
@@ -30,6 +34,14 @@ class UnknownNameError(Exception):
         return repr(self.name)
 
 
+class InvalidNameError(Exception):
+    def __init__(self, name):
+        self.name = name
+
+    def __str__(self):
+        return repr(self.name)
+
+
 class UnknownStreamTypeError(Exception):
     def __init__(self, stream_type):
         self.stream_type = stream_type

lbrynet/core/LBRYMetadata.py

@@ -0,0 +1,128 @@
+import requests
+import json
+import time
+
+from copy import deepcopy
+from googlefinance import getQuotes
+from lbrynet.conf import CURRENCIES
+from lbrynet.core import utils
+import logging
+
+log = logging.getLogger(__name__)
+
+BITTREX_FEE = 0.0025
+
+# Metadata version
+SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
+NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-'
+BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources']
+OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey']
+
+MV001 = "0.0.1"
+MV002 = "0.0.2"
+CURRENT_METADATA_VERSION = MV002
+
+METADATA_REVISIONS = {}
+METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}
+METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']}
+
+# Fee version
+BASE_FEE_FIELDS = ['amount', 'address']
+
+FV001 = "0.0.1"
+CURRENT_FEE_REVISION = FV001
+
+FEE_REVISIONS = {}
+FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []}
+
+
+def verify_name_characters(name):
+    for c in name:
+        assert c in NAME_ALLOWED_CHARSET, "Invalid character"
+    return True
+
+
+class LBRYFeeValidator(dict):
+    def __init__(self, fee_dict):
+        dict.__init__(self)
+        assert len(fee_dict) == 1
+        self.fee_version = None
+        self.currency_symbol = None
+
+        fee_to_load = deepcopy(fee_dict)
+
+        for currency in fee_dict:
+            self._verify_fee(currency, fee_to_load)
+
+        self.amount = self._get_amount()
+        self.address = self[self.currency_symbol]['address']
+
+    def _get_amount(self):
+        amt = self[self.currency_symbol]['amount']
+        if isinstance(amt, float):
+            return amt
+        else:
+            try:
+                return float(amt)
+            except TypeError:
+                log.error('Failed to convert %s to float', amt)
+                raise
+
+    def _verify_fee(self, currency, f):
+        # str in case someone made a claim with a wierd fee
+        assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency)
+        self.currency_symbol = currency
+        self.update({currency: {}})
+        for version in FEE_REVISIONS:
+            self._load_revision(version, f)
+            if not f:
+                self.fee_version = version
+                break
+        assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys())
+
+    def _load_revision(self, version, f):
+        for k in FEE_REVISIONS[version]['required']:
+            assert k in f[self.currency_symbol], "Missing required fee field: %s" % k
+            self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
+        for k in FEE_REVISIONS[version]['optional']:
+            if k in f[self.currency_symbol]:
+                self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
+
+
+class Metadata(dict):
+    def __init__(self, metadata):
+        dict.__init__(self)
+        self.meta_version = None
+        metadata_to_load = deepcopy(metadata)
+
+        self._verify_sources(metadata_to_load)
+        self._verify_metadata(metadata_to_load)
+
+    def _load_revision(self, version, metadata):
+        for k in METADATA_REVISIONS[version]['required']:
+            assert k in metadata, "Missing required metadata field: %s" % k
+            self.update({k: metadata.pop(k)})
+        for k in METADATA_REVISIONS[version]['optional']:
+            if k == 'fee':
+                self._load_fee(metadata)
+            elif k in metadata:
+                self.update({k: metadata.pop(k)})
+
+    def _load_fee(self, metadata):
+        if 'fee' in metadata:
+            self['fee'] = LBRYFeeValidator(metadata.pop('fee'))
+
+    def _verify_sources(self, metadata):
+        assert "sources" in metadata, "No sources given"
+        for source in metadata['sources']:
+            assert source in SOURCE_TYPES, "Unknown source type"
+
+    def _verify_metadata(self, metadata):
+        for version in METADATA_REVISIONS:
+            self._load_revision(version, metadata)
+            if not metadata:
+                self.meta_version = version
+                if utils.version_is_greater_than(self.meta_version, "0.0.1"):
+                    assert self.meta_version == self['ver'], "version mismatch"
+                break
+        assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys())

lbrynet/core/LBRYWallet.py

@@ -1,16 +1,12 @@
 import sys
-from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
+import datetime
-from lbrynet.core.client.ClientRequest import ClientRequest
+import logging
-from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
+import json
-from lbrynet.core.Error import InsufficientFundsError
+import subprocess
-from lbrynet.core.sqlite_helpers import rerun_if_locked
+import socket
-from lbrynet.conf import BASE_METADATA_FIELDS, SOURCE_TYPES, OPTIONAL_METADATA_FIELDS
+import time
-
+import os
-from lbryum import SimpleConfig, Network
+import requests
-from lbryum.lbrycrd import COIN, TYPE_ADDRESS
-from lbryum.wallet import WalletStorage, Wallet
-from lbryum.commands import known_commands, Commands
-from lbryum.transaction import Transaction
 
 from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
 from twisted.internet import threads, reactor, defer, task
@@ -19,13 +15,21 @@ from twisted.enterprise import adbapi
 from collections import defaultdict, deque
 from zope.interface import implements
 from decimal import Decimal
-import datetime
+from googlefinance import getQuotes
-import logging
+
-import json
+from lbryum import SimpleConfig, Network
-import subprocess
+from lbryum.lbrycrd import COIN, TYPE_ADDRESS
-import socket
+from lbryum.wallet import WalletStorage, Wallet
-import time
+from lbryum.commands import known_commands, Commands
-import os
+from lbryum.transaction import Transaction
+
+from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
+from lbrynet.core.client.ClientRequest import ClientRequest
+from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
+from lbrynet.core.Error import InsufficientFundsError
+from lbrynet.core.sqlite_helpers import rerun_if_locked
+from lbrynet.conf import SOURCE_TYPES
+from lbrynet.core.LBRYMetadata import Metadata
 
 log = logging.getLogger(__name__)
 alert = logging.getLogger("lbryalert." + __name__)
@@ -97,6 +101,7 @@ class LBRYWallet(object):
     def stop(self):
 
         self.stopped = True
+            
         # If self.next_manage_call is None, then manage is currently running or else
         # start has not been called, so set stopped and do nothing else.
         if self.next_manage_call is not None:
@@ -315,54 +320,66 @@ class LBRYWallet(object):
         return d
 
     def _get_stream_info_from_value(self, result, name):
-        r_dict = {}
+        def _check_result_fields(r):
-        if 'value' in result:
+            for k in ['value', 'txid', 'n', 'height', 'amount']:
-            value = result['value']
+                assert k in r, "getvalueforname response missing field %s" % k
 
-            try:
+        if 'error' in result:
-                value_dict = json.loads(value)
-            except (ValueError, TypeError):
-                return Failure(InvalidStreamInfoError(name))
-            r_dict['sources'] = value_dict['sources']
-            for field in BASE_METADATA_FIELDS:
-                    r_dict[field] = value_dict[field]
-            for field in value_dict:
-                if field in OPTIONAL_METADATA_FIELDS:
-                    r_dict[field] = value_dict[field]
-
-        if 'txid' in result:
-            d = self._save_name_metadata(name, r_dict['sources']['lbry_sd_hash'], str(result['txid']))
-            d.addCallback(lambda _: r_dict)
-            return d
-        elif 'error' in result:
             log.warning("Got an error looking up a name: %s", result['error'])
             return Failure(UnknownNameError(name))
-        else:
+
-            log.warning("Got an error looking up a name: %s", json.dumps(result))
+        _check_result_fields(result)
+
+        try:
+            metadata = Metadata(json.loads(result['value']))
+        except (ValueError, TypeError):
+            return Failure(InvalidStreamInfoError(name))
+
+        d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash'])
+        d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version)))
+        d.addCallback(lambda _: metadata)
+        return d
+
+    def _get_claim_info(self, result, name):
+        def _check_result_fields(r):
+            for k in ['value', 'txid', 'n', 'height', 'amount']:
+                assert k in r, "getvalueforname response missing field %s" % k
+
+        def _build_response(m, result):
+            result['value'] = m
+            return result
+
+        if 'error' in result:
+            log.warning("Got an error looking up a name: %s", result['error'])
             return Failure(UnknownNameError(name))
 
-    def claim_name(self, name, bid, sources, metadata, fee=None):
+        _check_result_fields(result)
-        value = {'sources': {}}
-        for k in SOURCE_TYPES:
-            if k in sources:
-                value['sources'][k] = sources[k]
-        if value['sources'] == {}:
-            return defer.fail("No source given")
-        for k in BASE_METADATA_FIELDS:
-            if k not in metadata:
-                return defer.fail("Missing required field '%s'" % k)
-            value[k] = metadata[k]
-        for k in metadata:
-            if k not in BASE_METADATA_FIELDS:
-                value[k] = metadata[k]
-        if fee is not None:
-            if "LBC" in fee:
-                value['fee'] = {'LBC': {'amount': fee['LBC']['amount'], 'address': fee['LBC']['address']}}
 
-        d = self._send_name_claim(name, json.dumps(value), bid)
+        try:
+            metadata = Metadata(json.loads(result['value']))
+        except (ValueError, TypeError):
+            return Failure(InvalidStreamInfoError(name))
+
+        d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash'])
+        d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version)))
+        d.addCallback(lambda _: _build_response(metadata, result))
+        return d
+
+    def get_claim_info(self, name):
+        d = self._get_value_for_name(name)
+        d.addCallback(lambda r: self._get_claim_info(r, name))
+        return d
+
+
+    def claim_name(self, name, bid, m):
+
+        metadata = Metadata(m)
+
+        d = self._send_name_claim(name, json.dumps(metadata), bid)
 
         def _save_metadata(txid):
-            d = self._save_name_metadata(name, value['sources']['lbry_sd_hash'], txid)
+            log.info("Saving metadata for claim %s" % txid)
+            d = self._save_name_metadata(name, txid, metadata['sources']['lbry_sd_hash'])
             d.addCallback(lambda _: txid)
             return d
 
@@ -407,10 +424,12 @@ class LBRYWallet(object):
         d.addCallback(self._get_decoded_tx)
         return d
 
-    def update_name(self, name, value, amount):
+    def update_name(self, name, bid, value, old_txid):
         d = self._get_value_for_name(name)
-        d.addCallback(lambda r: (self._update_name(r['txid'], json.dumps(value), amount), r['txid']))
+        d.addCallback(lambda r: self.abandon_name(r['txid'] if not old_txid else old_txid))
-        d.addCallback(lambda (new_txid, old_txid): self._update_name_metadata(name, value['sources']['lbry_sd_hash'], old_txid, new_txid))
+        d.addCallback(lambda r: log.info("Abandon claim tx %s" % str(r)))
+        d.addCallback(lambda _: self.claim_name(name, bid, value))
+
         return d
 
     def get_name_and_validity_for_sd_hash(self, sd_hash):
@@ -520,19 +539,13 @@ class LBRYWallet(object):
                                 "    txid text, " +
                                 "    sd_hash text)")
 
-    def _save_name_metadata(self, name, sd_hash, txid):
+    def _save_name_metadata(self, name, txid, sd_hash):
-        d = self.db.runQuery("select * from name_metadata where txid=?", (txid,))
+        d = self.db.runQuery("select * from name_metadata where name=? and txid=? and sd_hash=?", (name, txid, sd_hash))
         d.addCallback(lambda r: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, txid, sd_hash))
                                 if not len(r) else None)
 
         return d
 
-    def _update_name_metadata(self, name, sd_hash, old_txid, new_txid):
-        d = self.db.runQuery("delete * from name_metadata where txid=? and sd_hash=?", (old_txid, sd_hash))
-        d.addCallback(lambda _: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, new_txid, sd_hash)))
-        d.addCallback(lambda _: new_txid)
-        return d
-
     def _get_claim_metadata_for_sd_hash(self, sd_hash):
         d = self.db.runQuery("select name, txid from name_metadata where sd_hash=?", (sd_hash,))
         d.addCallback(lambda r: r[0] if len(r) else None)

lbrynet/lbrynet_daemon/LBRYDaemon.py

@@ -1,6 +1,4 @@
 import binascii
-from datetime import datetime
-from decimal import Decimal
 import distutils.version
 import locale
 import logging.handlers
@@ -13,18 +11,21 @@ import socket
 import string
 import subprocess
 import sys
-from urllib2 import urlopen
-
-from appdirs import user_data_dir
 import base58
 import requests
 import simplejson as json
+import pkg_resources
+
+from urllib2 import urlopen
+from appdirs import user_data_dir
+from datetime import datetime
+from decimal import Decimal
 from twisted.web import server
 from twisted.internet import defer, threads, error, reactor
 from twisted.internet.task import LoopingCall
 from txjsonrpc import jsonrpclib
 from txjsonrpc.web import jsonrpc
-from txjsonrpc.web.jsonrpc import Handler
+from txjsonrpc.web.jsonrpc import Handler, Proxy
 
 from lbrynet import __version__ as lbrynet_version
 from lbryum.version import LBRYUM_VERSION as lbryum_version
@@ -32,19 +33,21 @@ from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
 from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
 from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
-from lbrynet.core.Error import UnknownNameError, InsufficientFundsError
+from lbrynet.core.Error import UnknownNameError, InsufficientFundsError, InvalidNameError
 from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
 from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory
 from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
 from lbrynet.lbrynet_daemon.LBRYUIManager import LBRYUIManager
 from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream
 from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
+from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager
 from lbrynet.core import utils
+from lbrynet.core.LBRYMetadata import verify_name_characters
 from lbrynet.core.utils import generate_id
 from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
 from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \
-    DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, \
+    DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES
-    BASE_METADATA_FIELDS, OPTIONAL_METADATA_FIELDS, SOURCE_TYPES
+from lbrynet.conf import SEARCH_SERVERS
 from lbrynet.conf import DEFAULT_TIMEOUT, WALLET_TYPES
 from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob
 from lbrynet.core.Session import LBRYSession
@@ -68,7 +71,6 @@ lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
 
 log = logging.getLogger(__name__)
 
-
 if os.path.isfile(lbrynet_log):
     with open(lbrynet_log, 'r') as f:
         PREVIOUS_LBRYNET_LOG = len(f.read())
@@ -157,8 +159,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.current_db_revision = 1
         self.run_server = True
         self.session = None
+        self.exchange_rate_manager = ExchangeRateManager()
         self.waiting_on = {}
         self.streams = {}
+        self.pending_claims = {}
         self.known_dht_nodes = KNOWN_DHT_NODES
         self.first_run_after_update = False
         self.uploaded_temp_files = []
@@ -242,6 +246,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             self.session_settings['last_version'] = self.default_settings['last_version']
             self.first_run_after_update = True
             log.info("First run after update")
+            log.info("lbrynet %s --> %s" % (self.session_settings['last_version']['lbrynet'], self.default_settings['last_version']['lbrynet']))
+            log.info("lbryum %s --> %s" % (self.session_settings['last_version']['lbryum'], self.default_settings['last_version']['lbryum']))
 
         f = open(self.daemon_conf, "w")
         f.write(json.dumps(self.session_settings))
@@ -347,6 +353,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.internet_connection_checker = LoopingCall(self._check_network_connection)
         self.version_checker = LoopingCall(self._check_remote_versions)
         self.connection_problem_checker = LoopingCall(self._check_connection_problems)
+        self.pending_claim_checker = LoopingCall(self._check_pending_claims)
         # self.lbrynet_connection_checker = LoopingCall(self._check_lbrynet_connection)
 
         self.sd_identifier = StreamDescriptorIdentifier()
@@ -412,6 +419,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         return server.NOT_DONE_YET
 
     def _cbRender(self, result, request, id, version):
+        def default_decimal(obj):
+            if isinstance(obj, Decimal):
+                return float(obj)
+
         if isinstance(result, Handler):
             result = result.result
 
@@ -423,7 +434,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                 result = (result,)
             # Convert the result (python) to JSON-RPC
         try:
-            s = jsonrpclib.dumps(result, version=version)
+            s = jsonrpclib.dumps(result, version=version, default=default_decimal)
         except:
             f = jsonrpclib.Fault(self.FAILURE, "can't serialize output")
             s = jsonrpclib.dumps(f, version=version)
@@ -480,6 +491,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.internet_connection_checker.start(3600)
         self.version_checker.start(3600 * 12)
         self.connection_problem_checker.start(1)
+        self.exchange_rate_manager.start()
+
         if host_ui:
             self.lbry_ui_manager.update_checker.start(1800, now=False)
 
@@ -603,6 +616,47 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         if not self.connected_to_internet:
             self.connection_problem = CONNECTION_PROBLEM_CODES[1]
 
+    def _add_to_pending_claims(self, name, txid):
+        log.info("Adding lbry://%s to pending claims, txid %s" % (name, txid))
+        self.pending_claims[name] = txid
+        return txid
+
+    def _check_pending_claims(self):
+        # TODO: this was blatantly copied from jsonrpc_start_lbry_file. Be DRY.
+        def _start_file(f):
+            d = self.lbry_file_manager.toggle_lbry_file_running(f)
+            return defer.succeed("Started LBRY file")
+
+        def _get_and_start_file(name):
+            d = defer.succeed(self.pending_claims.pop(name))
+            d.addCallback(lambda _: self._get_lbry_file("name", name, return_json=False))
+            d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running")
+
+        def re_add_to_pending_claims(name):
+            txid = self.pending_claims.pop(name)
+            self._add_to_pending_claims(name, txid)
+
+        def _process_lbry_file(name, lbry_file):
+            # lbry_file is an instance of ManagedLBRYFileDownloader or None
+            # TODO: check for sd_hash in addition to txid
+            ready_to_start = (
+                lbry_file and
+                self.pending_claims[name] == lbry_file.txid
+            )
+            if ready_to_start:
+                _get_and_start_file(name)
+            else:
+                re_add_to_pending_claims(name)
+
+        for name in self.pending_claims:
+            log.info("Checking if new claim for lbry://%s is confirmed" % name)
+            d = self._resolve_name(name, force_refresh=True)
+            d.addCallback(lambda _: self._get_lbry_file_by_uri(name))
+            d.addCallbacks(
+                lambda lbry_file: _process_lbry_file(name, lbry_file),
+                lambda _: re_add_to_pending_claims(name)
+            )
+
     def _start_server(self):
         if self.peer_port is not None:
 
@@ -721,6 +775,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             self.connection_problem_checker.stop()
         if self.lbry_ui_manager.update_checker.running:
             self.lbry_ui_manager.update_checker.stop()
+        if self.pending_claim_checker.running:
+            self.pending_claim_checker.stop()
 
         self._clean_up_temp_files()
 
@@ -1089,8 +1145,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
                     return defer.succeed(None)
 
             self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet,
-                                           self.lbry_file_manager, max_key_fee=self.max_key_fee,
+                                           self.lbry_file_manager, self.exchange_rate_manager,
-                                           data_rate=self.data_rate, timeout=timeout,
+                                           max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout,
                                            download_directory=download_directory, file_name=file_name)
             d = self.streams[name].start(stream_info, name)
             if wait_for_write:
@@ -1120,6 +1176,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         return defer.succeed(True)
 
     def _resolve_name(self, name, force_refresh=False):
+        try:
+            verify_name_characters(name)
+        except AssertionError:
+            log.error("Bad name")
+            return defer.fail(InvalidNameError("Bad name"))
+
         def _cache_stream_info(stream_info):
             def _add_txid(txid):
                 self.name_cache[name]['txid'] = txid
@@ -1186,7 +1248,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         def _add_key_fee(data_cost):
             d = self._resolve_name(name)
-            d.addCallback(lambda info: data_cost if 'fee' not in info else data_cost + info['fee']['LBC']['amount'])
+            d.addCallback(lambda info: self.exchange_rate_manager.to_lbc(info.get('fee', None)))
+            d.addCallback(lambda fee: data_cost if fee is None else data_cost + fee.amount)
             return d
 
         d = self._resolve_name(name)
@@ -1196,8 +1259,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
         d.addCallback(lambda metadata: metadata.validator.info_to_show())
         d.addCallback(lambda info: int(dict(info)['stream_size']) / 1000000 * self.data_rate)
-        d.addCallback(_add_key_fee)
+        d.addCallbacks(_add_key_fee, lambda _: _add_key_fee(0.0))
-        d.addErrback(lambda _: _add_key_fee(0.0))
         reactor.callLater(self.search_timeout, _check_est, d, name)
 
         return d
@@ -1305,7 +1367,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
                 if f.txid:
                     d = self._resolve_name(f.uri)
-                    d.addCallback(_add_to_dict)
+                    d.addCallbacks(_add_to_dict, lambda _: _add_to_dict("Pending confirmation"))
                 else:
                     d = defer.succeed(message)
                 return d
@@ -1363,6 +1425,10 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
         return defer.succeed(None)
 
+    def _search(self, search):
+        proxy = Proxy(random.choice(SEARCH_SERVERS))
+        return proxy.callRemote('search', search)
+
     def _render_response(self, result, code):
         return defer.succeed({'result': result, 'code': code})
 
@@ -1482,10 +1548,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             'ui_version': self.ui_version,
             'remote_lbrynet': self.git_lbrynet_version,
             'remote_lbryum': self.git_lbryum_version,
-            'lbrynet_update_available': utils.version_is_greater_than(
+            'lbrynet_update_available': utils.version_is_greater_than(self.git_lbrynet_version, lbrynet_version),
-                self.git_lbrynet_version, lbrynet_version),
+            'lbryum_update_available': utils.version_is_greater_than(self.git_lbryum_version, lbryum_version),
-            'lbryum_update_available': utils.version_is_greater_than(
-                self.git_lbryum_version, lbryum_version)
         }
 
         log.info("Get version info: " + json.dumps(msg))
@@ -1667,15 +1731,37 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             metadata from name claim
         """
 
-        if 'name' in p.keys():
+        force = p.get('force', False)
+
+        if 'name' in p:
             name = p['name']
         else:
             return self._render_response(None, BAD_REQUEST)
 
-        d = self._resolve_name(name)
+        d = self._resolve_name(name, force_refresh=force)
         d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure)
         return d
 
+    def jsonrpc_get_claim_info(self, p):
+        """
+            Resolve claim info from a LBRY uri
+
+            Args:
+                'name': name to look up, string, do not include lbry:// prefix
+            Returns:
+                txid, amount, value, n, height
+        """
+
+        def _convert_amount_to_float(r):
+            r['amount'] = float(r['amount']) / 10**8
+            return r
+
+        name = p['name']
+        d = self.session.wallet.get_claim_info(name)
+        d.addCallback(_convert_amount_to_float)
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
     def jsonrpc_get(self, p):
         """
         Download stream from a LBRY uri
@@ -1786,9 +1872,24 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda r: self._render_response(r, OK_CODE))
         return d
 
+    def jsonrpc_get_est_cost(self, p):
+        """
+        Get estimated cost for a lbry uri
+
+        Args:
+            'name': lbry uri
+        Returns:
+            estimated cost
+        """
+
+        name = p['name']
+        d = self._get_est_cost(name)
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        return d
+
     def jsonrpc_search_nametrie(self, p):
         """
-        Search the nametrie for claims beginning with search (yes, this is a dumb search, it'll be made better)
+        Search the nametrie for claims
 
         Args:
             'search': search query, string
@@ -1796,6 +1897,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             List of search results
         """
 
+        # TODO: change this function to "search", and use cached stream size info from the search server
+
         if 'search' in p.keys():
             search = p['search']
         else:
@@ -1805,44 +1908,31 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             t = []
             for i in n:
                 if i[0]:
-                    if i[1][0][0] and i[1][1][0] and i[1][2][0]:
+                    tr = {}
-                        i[1][0][1]['value'] = str(i[1][0][1]['value'])
+                    tr.update(i[1][0]['value'])
-                        t.append([i[1][0][1], i[1][1][1], i[1][2][1]])
+                    thumb = tr.get('thumbnail', None)
+                    if thumb is None:
+                        tr['thumbnail'] = "img/Free-speech-flag.svg"
+                    tr['name'] = i[1][0]['name']
+                    tr['cost_est'] = i[1][1]
+                    t.append(tr)
             return t
 
-        def resolve_claims(claims):
+        def get_est_costs(results):
-            ds = []
+            def _save_cost(search_result):
-            for claim in claims:
+                d = self._get_est_cost(search_result['name'])
-                d1 = defer.succeed(claim)
+                d.addCallback(lambda p: [search_result, p])
-                d2 = self._resolve_name(claim['name'])
+                return d
-                d3 = self._get_est_cost(claim['name'])
-                dl = defer.DeferredList([d1, d2, d3], consumeErrors=True)
-                ds.append(dl)
-            return defer.DeferredList(ds)
 
-        def _disp(results):
+            dl = defer.DeferredList([_save_cost(r) for r in results], consumeErrors=True)
-            log.info('Found ' + str(len(results)) + ' search results')
+            return dl
-            consolidated_results = []
-            for r in results:
-                t = {}
-                t.update(r[0])
-                if not 'thumbnail' in r[1].keys():
-                    r[1]['thumbnail'] = "img/Free-speech-flag.svg"
-                t.update(r[1])
-                t['cost_est'] = r[2]
-                consolidated_results.append(t)
-                # log.info(str(t))
 
-            return consolidated_results
+        log.info('Search: %s' % search)
 
-        log.info('Search nametrie: ' + search)
+        d = self._search(search)
-
-        d = self.session.wallet.get_nametrie()
-        d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim])
         d.addCallback(lambda claims: claims[:self.max_search_results])
-        d.addCallback(resolve_claims)
+        d.addCallback(get_est_costs)
         d.addCallback(_clean)
-        d.addCallback(_disp)
         d.addCallback(lambda results: self._render_response(results, OK_CODE))
 
         return d
@@ -1889,21 +1979,49 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         Returns:
             Claim txid
         """
-        # start(self, name, file_path, bid, metadata, fee=None, sources=None):
+
         name = p['name']
+        try:
+            verify_name_characters(name)
+        except:
+            log.error("Bad name")
+            return defer.fail(InvalidNameError("Bad name"))
         bid = p['bid']
         file_path = p['file_path']
         metadata = p['metadata']
+
+        def _set_address(address, currency):
+            log.info("Generated new address for key fee: " + str(address))
+            metadata['fee'][currency]['address'] = address
+            return defer.succeed(None)
+
+        def _delete_data(lbry_file):
+            txid = lbry_file.txid
+            d = self._delete_lbry_file(lbry_file, delete_file=False)
+            d.addCallback(lambda _: txid)
+            return d
+
+        if not self.pending_claim_checker.running:
+            self.pending_claim_checker.start(30)
+
+        d = self._resolve_name(name, force_refresh=True)
+        d.addErrback(lambda _: None)
+
         if 'fee' in p:
-            fee = p['fee']
+            metadata['fee'] = p['fee']
-        else:
+            assert len(metadata['fee']) == 1, "Too many fees"
-            fee = None
+            for c in metadata['fee']:
+                if 'address' not in metadata['fee'][c]:
+                    d.addCallback(lambda _: self.session.wallet.get_new_address())
+                    d.addCallback(lambda addr: _set_address(addr, c))
 
         pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet)
-
+        d.addCallback(lambda _: self._get_lbry_file_by_uri(name))
-        d = pub.start(name, file_path, bid, metadata, fee)
+        d.addCallbacks(lambda l: None if not l else _delete_data(l), lambda _: None)
-        d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE),
+        d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r))
-                       lambda err: self._render_response(err.getTraceback(), BAD_REQUEST))
+        d.addCallback(lambda txid: self._add_to_pending_claims(name, txid))
+        d.addCallback(lambda r: self._render_response(r, OK_CODE))
+        d.addErrback(lambda err: self._render_response(err.getTraceback(), BAD_REQUEST))
 
         return d
 
@@ -2166,27 +2284,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda r: self._render_response(r, OK_CODE))
         return d
 
-    def jsonrpc_update_name(self, p):
-        """
-        Update name claim
-
-        Args:
-            'name': the uri of the claim to be updated
-            'metadata': new metadata dict
-            'amount': bid amount of updated claim
-        Returns:
-            txid
-        """
-
-        name = p['name']
-        metadata = p['metadata'] if isinstance(p['metadata'], dict) else json.loads(p['metadata'])
-        amount = p['amount']
-
-        d = self.session.wallet.update_name(name, metadata, amount)
-        d.addCallback(lambda r: self._render_response(r, OK_CODE))
-
-        return d
-
     def jsonrpc_log(self, p):
         """
         Log message
@@ -2282,7 +2379,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
             # No easy way to reveal specific files on Linux, so just open the containing directory
             d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.dirname(path)])
 
-
         d.addCallback(lambda _: self._render_response(True, OK_CODE))
         return d
 

lbrynet/lbrynet_daemon/LBRYDownloader.py

@@ -3,14 +3,16 @@ import logging
 import os
 import sys
 
+from copy import deepcopy
 from appdirs import user_data_dir
 from datetime import datetime
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
 
-from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError
+from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError, KeyFeeAboveMaxAllowed
 from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.core.StreamDescriptor import download_sd_blob
+from lbrynet.core.LBRYMetadata import Metadata, LBRYFeeValidator
 from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
 from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME
 
@@ -40,17 +42,17 @@ log = logging.getLogger(__name__)
 
 
 class GetStream(object):
-    def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, data_rate=0.5,
+    def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager,
-                                    timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None):
+                 max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None):
         self.wallet = wallet
         self.resolved_name = None
         self.description = None
-        self.key_fee = None
+        self.fee = None
-        self.key_fee_address = None
         self.data_rate = data_rate
         self.name = None
         self.file_name = file_name
         self.session = session
+        self.exchange_rate_manager = exchange_rate_manager
         self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
         self.lbry_file_manager = lbry_file_manager
         self.sd_identifier = sd_identifier
@@ -64,7 +66,7 @@ class GetStream(object):
         self.download_directory = download_directory
         self.download_path = None
         self.downloader = None
-        self.finished = defer.Deferred()
+        self.finished = defer.Deferred(None)
         self.checker = LoopingCall(self.check_status)
         self.code = STREAM_STAGES[0]
 
@@ -83,30 +85,16 @@ class GetStream(object):
             self.code = STREAM_STAGES[4]
             self.finished.callback(False)
 
-    def start(self, stream_info, name):
+    def _convert_max_fee(self):
-        self.resolved_name = name
+        if isinstance(self.max_key_fee, dict):
-        self.stream_info = stream_info
+            max_fee = LBRYFeeValidator(self.max_key_fee)
-        if 'sources' in self.stream_info:
+            if max_fee.currency_symbol == "LBC":
-            self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
+                return max_fee.amount
-        else:
+            return self.exchange_rate_manager.to_lbc(self.fee).amount
-            raise InvalidStreamInfoError(self.stream_info)
+        elif isinstance(self.max_key_fee, float):
-        if 'description' in self.stream_info:
+            return float(self.max_key_fee)
-            self.description = self.stream_info['description']
-        if 'fee' in self.stream_info:
-            if 'LBC' in self.stream_info['fee']:
-                self.key_fee = float(self.stream_info['fee']['LBC']['amount'])
-                self.key_fee_address = self.stream_info['fee']['LBC']['address']
-            else:
-                self.key_fee_address = None
-        else:
-            self.key_fee = None
-            self.key_fee_address = None
-        if self.key_fee > self.max_key_fee:
-            log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name))
-            return defer.fail(None)
-        else:
-            pass
 
+    def start(self, stream_info, name):
         def _cause_timeout(err):
             log.error(err)
             log.debug('Forcing a timeout')
@@ -131,6 +119,23 @@ class GetStream(object):
                                            download_directory=self.download_directory,
                                            file_name=self.file_name)
 
+        self.resolved_name = name
+        self.stream_info = deepcopy(stream_info)
+        self.description = self.stream_info['description']
+        self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
+
+        if 'fee' in self.stream_info:
+            self.fee = LBRYFeeValidator(self.stream_info['fee'])
+            max_key_fee = self._convert_max_fee()
+            if self.exchange_rate_manager.to_lbc(self.fee).amount > max_key_fee:
+                log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount,
+                                                                                     self.max_key_fee,
+                                                                                     self.resolved_name))
+                return defer.fail(KeyFeeAboveMaxAllowed())
+            log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee),
+                                                                              max_key_fee,
+                                                                              self.resolved_name))
+
         self.checker.start(1)
 
         self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
@@ -146,17 +151,20 @@ class GetStream(object):
 
     def _start_download(self, downloader):
         def _pay_key_fee():
-            if self.key_fee is not None and self.key_fee_address is not None:
+            if self.fee is not None:
-                reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee)
+                fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount
+                reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc)
                 if reserved_points is None:
                     return defer.fail(InsufficientFundsError())
-                log.info("Key fee: %f --> %s" % (self.key_fee, self.key_fee_address))
+                return self.wallet.send_points_to_address(reserved_points, fee_lbc)
-                return self.wallet.send_points_to_address(reserved_points, self.key_fee)
+
             return defer.succeed(None)
 
         d = _pay_key_fee()
+
         self.downloader = downloader
         self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
+
         d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name))
         d.addCallback(lambda _: self.downloader.start())
 

lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py

@@ -0,0 +1,215 @@
+import time
+import requests
+import logging
+import json
+import googlefinance
+from twisted.internet import defer, reactor
+from twisted.internet.task import LoopingCall
+
+from lbrynet.core.LBRYMetadata import LBRYFeeValidator
+
+log = logging.getLogger(__name__)
+
+CURRENCY_PAIRS = ["USDBTC", "BTCLBC"]
+BITTREX_FEE = 0.0025
+COINBASE_FEE = 0.0 #add fee
+
+
+class ExchangeRate(object):
+    def __init__(self, market, spot, ts):
+        assert int(time.time()) - ts < 600
+        self.currency_pair = (market[0:3], market[3:6])
+        self.spot = spot
+        self.ts = ts
+
+    def as_dict(self):
+        return {'spot': self.spot, 'ts': self.ts}
+
+
+class MarketFeed(object):
+    def __init__(self, market, name, url, params, fee):
+        self.market = market
+        self.name = name
+        self.url = url
+        self.params = params
+        self.fee = fee
+        self.rate = None
+        self._updater = LoopingCall(self._update_price)
+
+    def _make_request(self):
+        r = requests.get(self.url, self.params)
+        return r.text
+
+    def _handle_response(self, response):
+        return NotImplementedError
+
+    def _subtract_fee(self, from_amount):
+        return defer.succeed(from_amount / (1.0 - self.fee))
+
+    def _save_price(self, price):
+        log.info("Saving price update %f for %s" % (price, self.market))
+        self.rate = ExchangeRate(self.market, price, int(time.time()))
+
+    def _update_price(self):
+        d = defer.succeed(self._make_request())
+        d.addCallback(self._handle_response)
+        d.addCallback(self._subtract_fee)
+        d.addCallback(self._save_price)
+
+    def start(self):
+        if not self._updater.running:
+            self._updater.start(300)
+
+    def stop(self):
+        if self._updater.running:
+            self._updater.stop()
+
+
+class BittrexFeed(MarketFeed):
+    def __init__(self):
+        MarketFeed.__init__(
+            self,
+            "BTCLBC",
+            "Bittrex",
+            "https://bittrex.com/api/v1.1/public/getmarkethistory",
+            {'market': 'BTC-LBC', 'count': 50},
+            BITTREX_FEE
+        )
+
+    def _handle_response(self, response):
+        trades = json.loads(response)['result']
+        vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades])
+        return defer.succeed(float(1.0 / vwap))
+
+
+class GoogleBTCFeed(MarketFeed):
+    def __init__(self):
+        MarketFeed.__init__(
+            self,
+            "USDBTC",
+            "Coinbase via Google finance",
+            None,
+            None,
+            COINBASE_FEE
+        )
+
+    def _make_request(self):
+        return googlefinance.getQuotes('CURRENCY:USDBTC')[0]
+
+    def _handle_response(self, response):
+        return float(response['LastTradePrice'])
+
+
+def get_default_market_feed(currency_pair):
+    currencies = None
+    if isinstance(currency_pair, str):
+        currencies = (currency_pair[0:3], currency_pair[3:6])
+    elif isinstance(currency_pair, tuple):
+        currencies = currency_pair
+    assert currencies is not None
+
+    if currencies == ("USD", "BTC"):
+        return GoogleBTCFeed()
+    elif currencies == ("BTC", "LBC"):
+        return BittrexFeed()
+
+
+class ExchangeRateManager(object):
+    def __init__(self):
+        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
+        self.market_feeds = [get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS]
+
+    def start(self):
+        log.info("Starting exchange rate manager")
+        for feed in self.market_feeds:
+            feed.start()
+
+    def stop(self):
+        log.info("Stopping exchange rate manager")
+        for source in self.market_feeds:
+            source.stop()
+
+    def convert_currency(self, from_currency, to_currency, amount):
+        log.info("Converting %f %s to %s" % (amount, from_currency, to_currency))
+        if from_currency == to_currency:
+            return amount
+        for market in self.market_feeds:
+            if market.rate.currency_pair == (from_currency, to_currency):
+                return amount * market.rate.spot
+        for market in self.market_feeds:
+            if market.rate.currency_pair[0] == from_currency:
+                return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
+        raise Exception('Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency))
+
+    def fee_dict(self):
+        return {market: market.rate.as_dict() for market in self.market_feeds}
+
+    def to_lbc(self, fee):
+        if fee is None:
+            return None
+        if not isinstance(fee, LBRYFeeValidator):
+            fee_in = LBRYFeeValidator(fee)
+        else:
+            fee_in = fee
+
+        return LBRYFeeValidator({fee_in.currency_symbol:
+                                    {
+                                        'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount),
+                                        'address': fee_in.address
+                                    }
+        })
+
+
+class DummyBTCLBCFeed(MarketFeed):
+    def __init__(self):
+        MarketFeed.__init__(
+            self,
+            "BTCLBC",
+            "market name",
+            "derp.com",
+            None,
+            0.0
+        )
+
+
+class DummyUSDBTCFeed(MarketFeed):
+    def __init__(self):
+        MarketFeed.__init__(
+            self,
+            "USDBTC",
+            "market name",
+            "derp.com",
+            None,
+            0.0
+        )
+
+
+class DummyExchangeRateManager(object):
+    def __init__(self, rates):
+        self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()]
+        for feed in self.market_feeds:
+            feed.rate = ExchangeRate(feed.market, rates[feed.market]['spot'], rates[feed.market]['ts'])
+
+    def convert_currency(self, from_currency, to_currency, amount):
+        log.info("Converting %f %s to %s" % (amount, from_currency, to_currency))
+        for market in self.market_feeds:
+            if market.rate.currency_pair == (from_currency, to_currency):
+                return amount * market.rate.spot
+        for market in self.market_feeds:
+            if market.rate.currency_pair[0] == from_currency:
+                return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
+
+    def to_lbc(self, fee):
+        if fee is None:
+            return None
+        if not isinstance(fee, LBRYFeeValidator):
+            fee_in = LBRYFeeValidator(fee)
+        else:
+            fee_in = fee
+
+        return LBRYFeeValidator({fee_in.currency_symbol:
+                                    {
+                                        'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount),
+                                        'address': fee_in.address
+                                    }
+        })

lbrynet/lbrynet_daemon/LBRYPublisher.py

@@ -10,6 +10,7 @@ from lbrynet.core.Error import InsufficientFundsError
 from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
 from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
 from lbrynet.core.PaymentRateManager import PaymentRateManager
+from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION
 from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
 from lbrynet.conf import LOG_FILE_NAME
 from twisted.internet import threads, defer
@@ -39,10 +40,10 @@ class Publisher(object):
         self.verified = False
         self.lbry_file = None
         self.txid = None
-        self.sources = {}
+        self.stream_hash = None
-        self.fee = None
+        self.metadata = {}
 
-    def start(self, name, file_path, bid, metadata, fee=None, sources={}):
+    def start(self, name, file_path, bid, metadata, old_txid):
 
         def _show_result():
             log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid)
@@ -51,8 +52,8 @@ class Publisher(object):
         self.publish_name = name
         self.file_path = file_path
         self.bid_amount = bid
-        self.fee = fee
         self.metadata = metadata
+        self.old_txid = old_txid
 
         d = self._check_file_path(self.file_path)
         d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
@@ -60,6 +61,7 @@ class Publisher(object):
         d.addCallback(self.add_to_lbry_files)
         d.addCallback(lambda _: self._create_sd_blob())
         d.addCallback(lambda _: self._claim_name())
+        d.addCallback(lambda _: self.set_status())
         d.addCallbacks(lambda _: _show_result(), self._show_publish_error)
 
         return d
@@ -72,26 +74,15 @@ class Publisher(object):
             return True
         return threads.deferToThread(check_file_threaded)
 
-    def _get_new_address(self):
+    def set_lbry_file(self, lbry_file_downloader):
-        d = self.wallet.get_new_address()
-
-        def set_address(address):
-            self.key_fee_address = address
-            return True
-
-        d.addCallback(set_address)
-        return d
-
-    def set_status(self, lbry_file_downloader):
         self.lbry_file = lbry_file_downloader
-        d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED)
+        return defer.succeed(None)
-        d.addCallback(lambda _: lbry_file_downloader.restore())
-        return d
 
     def add_to_lbry_files(self, stream_hash):
+        self.stream_hash = stream_hash
         prm = PaymentRateManager(self.session.base_payment_rate_manager)
         d = self.lbry_file_manager.add_lbry_file(stream_hash, prm)
-        d.addCallback(self.set_status)
+        d.addCallback(self.set_lbry_file)
         return d
 
     def _create_sd_blob(self):
@@ -99,19 +90,34 @@ class Publisher(object):
                             self.lbry_file.stream_hash)
 
         def set_sd_hash(sd_hash):
-            self.sources['lbry_sd_hash'] = sd_hash
+            if 'sources' not in self.metadata:
+                self.metadata['sources'] = {}
+            self.metadata['sources']['lbry_sd_hash'] = sd_hash
 
         d.addCallback(set_sd_hash)
         return d
 
+    def set_status(self):
+        d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED)
+        d.addCallback(lambda _: self.lbry_file.restore())
+        return d
+
     def _claim_name(self):
         self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory,
                                                                           self.lbry_file.file_name))[0]
+        self.metadata['ver'] = CURRENT_METADATA_VERSION
+
+        if self.old_txid:
+
+            d = self.wallet.abandon_name(self.old_txid)
+            d.addCallback(lambda tx: log.info("Abandoned tx %s" % str(tx)))
+            d.addCallback(lambda _: self.wallet.claim_name(self.publish_name,
+                                       self.bid_amount,
+                                       Metadata(self.metadata)))
+        else:
             d = self.wallet.claim_name(self.publish_name,
                                        self.bid_amount,
-                                   self.sources,
+                                       Metadata(self.metadata))
-                                   self.metadata,
-                                   fee=self.fee)
         def set_tx_hash(txid):
             self.txid = txid
 

requirements.txt

@@ -26,3 +26,4 @@ unqlite==0.2.0
 wsgiref==0.1.2
 zope.interface==4.1.3
 base58==0.2.2
+googlefinance==0.7

setup.py

@@ -25,7 +25,7 @@ console_scripts = ['lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUp
 
 requires = ['pycrypto', 'twisted', 'miniupnpc', 'yapsy', 'seccure',
             'python-bitcoinrpc==0.1', 'txJSON-RPC', 'requests>=2.4.2', 'unqlite==0.2.0',
-            'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58']
+            'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58', 'googlefinance']
 
 setup(name='lbrynet',
       description='A decentralized media library and marketplace',

tests/lbrynet/core/test_LBRYExchangeRateManager.py

@@ -0,0 +1,38 @@
+import mock
+from lbrynet.core import LBRYMetadata
+from lbrynet.lbrynet_daemon import LBRYExchangeRateManager
+
+from twisted.trial import unittest
+
+
+class LBRYFeeFormatTest(unittest.TestCase):
+    def test_fee_created_with_correct_inputs(self):
+        fee_dict = {
+            'USD': {
+                'amount': 10.0,
+                'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
+            }
+        }
+        fee = LBRYMetadata.LBRYFeeValidator(fee_dict)
+        self.assertEqual(10.0, fee['USD']['amount'])
+
+
+class LBRYFeeTest(unittest.TestCase):
+    def setUp(self):
+        self.patcher = mock.patch('time.time')
+        self.time = self.patcher.start()
+        self.time.return_value = 0
+
+    def tearDown(self):
+        self.time.stop()
+
+    def test_fee_converts_to_lbc(self):
+        fee_dict = {
+            'USD': {
+                'amount': 10.0,
+                'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
+            }
+        }
+        rates = {'BTCLBC': {'spot': 3.0, 'ts': 2}, 'USDBTC': {'spot': 2.0, 'ts': 3}}
+        manager = LBRYExchangeRateManager.DummyExchangeRateManager(rates)
+        self.assertEqual(60.0, manager.to_lbc(fee_dict).amount)

tests/lbrynet/core/test_LBRYMetadata.py

@@ -0,0 +1,129 @@
+from lbrynet.core import LBRYMetadata
+from twisted.trial import unittest
+
+
+class MetadataTest(unittest.TestCase):
+    def test_assertion_if_source_is_missing(self):
+        metadata = {}
+        with self.assertRaises(AssertionError):
+            LBRYMetadata.Metadata(metadata)
+
+    def test_metadata_works_without_fee(self):
+        metadata = {
+            'license': 'Oscilloscope Laboratories',
+            'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.',
+            'language': 'en',
+            'title': "It's a Disaster",
+            'author': 'Written and directed by Todd Berger',
+            'sources': {
+                'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'},
+            'content-type': 'audio/mpeg',
+            'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg'
+        }
+        m = LBRYMetadata.Metadata(metadata)
+        self.assertFalse('key' in m)
+
+    def test_assertion_if_invalid_source(self):
+        metadata = {
+            'license': 'Oscilloscope Laboratories',
+            'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}},
+            'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.',
+            'language': 'en',
+            'title': "It's a Disaster",
+            'author': 'Written and directed by Todd Berger',
+            'sources': {
+                'fake': 'source'},
+            'content-type': 'audio/mpeg',
+            'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg'
+        }
+        with self.assertRaises(AssertionError):
+            LBRYMetadata.Metadata(metadata)
+
+    def test_assertion_if_missing_v001_field(self):
+        metadata = {
+            'license': 'Oscilloscope Laboratories',
+            'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}},
+            'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.',
+            'language': 'en',
+            'author': 'Written and directed by Todd Berger',
+            'sources': {
+                'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'},
+            'content-type': 'audio/mpeg',
+            'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg'
+        }
+        with self.assertRaises(AssertionError):
+            LBRYMetadata.Metadata(metadata)
+
+    def test_version_is_001_if_all_fields_are_present(self):
+        metadata = {
+            'license': 'Oscilloscope Laboratories',
+            'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}},
+            'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.',
+            'language': 'en',
+            'title': "It's a Disaster",
+            'author': 'Written and directed by Todd Berger',
+            'sources': {
+                'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'},
+            'content-type': 'audio/mpeg',
+            'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg'
+        }
+        m = LBRYMetadata.Metadata(metadata)
+        self.assertEquals('0.0.1', m.meta_version)
+
+    def test_assertion_if_there_is_an_extra_field(self):
+        metadata = {
+            'license': 'NASA',
+            'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}},
+            'ver': '0.0.2',
+            'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines',
+            'language': 'en',
+            'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger',
+            'title': 'Thermonuclear Art',
+            'sources': {
+                        'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'},
+            'nsfw': False,
+            'content-type': 'video/mp4',
+            'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg',
+            'MYSTERYFIELD': '?'
+        }
+        with self.assertRaises(AssertionError):
+            LBRYMetadata.Metadata(metadata)
+
+    def test_version_is_002_if_all_fields_are_present(self):
+        metadata = {
+            'license': 'NASA',
+            'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}},
+            'ver': '0.0.2',
+            'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines',
+            'language': 'en',
+            'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger',
+            'title': 'Thermonuclear Art',
+            'sources': {
+                        'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'},
+            'nsfw': False,
+            'content-type': 'video/mp4',
+            'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg'
+        }
+        m = LBRYMetadata.Metadata(metadata)
+        self.assertEquals('0.0.2', m.meta_version)
+
+    def test_version_claimed_is_001_but_version_is_002(self):
+        metadata = {
+            'license': 'NASA',
+            'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}},
+            'ver': '0.0.1',
+            'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines',
+            'language': 'en',
+            'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger',
+            'title': 'Thermonuclear Art',
+            'sources': {
+                'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'},
+            'nsfw': False,
+            'content-type': 'video/mp4',
+            'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg'
+        }
+        with self.assertRaises(AssertionError):
+            LBRYMetadata.Metadata(metadata)
+
+
+