From 2dcd2b13befeff131e78c6208726b5b66ae4c722 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 15 Jul 2016 14:09:43 -0400 Subject: [PATCH 01/73] add LBRYMetadata.py --- lbrynet/core/LBRYMetadata.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lbrynet/core/LBRYMetadata.py diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py new file mode 100644 index 000000000..565a10785 --- /dev/null +++ b/lbrynet/core/LBRYMetadata.py @@ -0,0 +1,28 @@ +import json + +BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type'] +OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] + +#v0.0.1 metadata +METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}} + +#v0.0.2 metadata additions +METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw'], 'optional': []} + + +class Metadata(dict): + def __init__(self, metadata): + dict.__init__(self) + self.metaversion = None + m = metadata.copy() + for version in METADATA_REVISIONS: + for k in METADATA_REVISIONS[version]['required']: + assert k in metadata, "Missing required metadata field: %s" % k + self.update({k: m.pop(k)}) + for k in METADATA_REVISIONS[version]['optional']: + if k in metadata: + self.update({k: m.pop(k)}) + if not len(m): + self.metaversion = version + break + assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys()) \ No newline at end of file From 25d1b8b9e5a35ba7f9b3a2924a93544fbfa041fb Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 21 Jul 2016 16:11:14 -0400 Subject: [PATCH 02/73] move towards better fees and metadata --- lbrynet/conf.py | 9 +++-- lbrynet/core/LBRYFee.py | 39 +++++++++++++++++++ lbrynet/core/LBRYWallet.py | 7 ---- lbrynet/lbrynet_daemon/LBRYDaemon.py | 23 +++++++++-- lbrynet/lbrynet_daemon/LBRYDownloader.py | 10 ++--- .../daemon_scripts/Autofetcher.py | 22 +++++++++++ 6 files changed, 89 insertions(+), 21 deletions(-) create mode 100644 lbrynet/core/LBRYFee.py diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 9ead15f4d..59150a085 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -42,11 +42,14 @@ DEFAULT_WALLET = "lbryum" WALLET_TYPES = ["lbryum", "lbrycrd"] DEFAULT_TIMEOUT = 30 DEFAULT_MAX_SEARCH_RESULTS = 25 -DEFAULT_MAX_KEY_FEE = 100.0 +DEFAULT_MAX_KEY_FEE = {'BTC': {'amount': 0.025}} DEFAULT_SEARCH_TIMEOUT = 3.0 DEFAULT_CACHE_TIME = 3600 DEFAULT_UI_BRANCH = "master" SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] -BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type'] -OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] \ No newline at end of file + +CURRENCIES = [ + {'BTC': {'type': 'crypto'}}, + {'LBC': {'type': 'crypto'}}, +] \ No newline at end of file diff --git a/lbrynet/core/LBRYFee.py b/lbrynet/core/LBRYFee.py new file mode 100644 index 000000000..09b7dbd39 --- /dev/null +++ b/lbrynet/core/LBRYFee.py @@ -0,0 +1,39 @@ +import requests +import json + +from lbrynet.conf import CURRENCIES + + +class LBRYFee(object): + def __init__(self, currency, amount, address=None): + assert currency in [c.keys()[0] for c in CURRENCIES], "Unsupported currency: %s" % str(currency) + self.address = address + self.currency_symbol = currency + self.currency = [c for c in CURRENCIES if self.currency_symbol in c][0] + if not isinstance(amount, float): + self.amount = float(amount) + else: + self.amount = amount + + def convert_to(self, to_currency, rate_dict={}): + if to_currency is self.currency_symbol: + return self.as_dict() + if self.currency[self.currency_symbol]['type'] is 'fiat': + raise NotImplemented + else: + if to_currency not in rate_dict: + params = {'market': '%s-%s' % (self.currency_symbol, to_currency)} + r = requests.get("https://bittrex.com/api/v1.1/public/getticker", params) + last = json.loads(r.text)['result']['Last'] + converted = self.amount / float(last) + else: + converted = self.amount / float(rate_dict[to_currency]['last']) + + return LBRYFee(to_currency, converted, self.address).as_dict() + + def as_dict(self): + return {self.currency_symbol: {'amount': self.amount, 'address': self.address}} + + def from_dict(self, fee_dict): + s = fee_dict.keys()[0] + return LBRYFee(s, fee_dict[s]['amount'], fee_dict[s]['address']) \ No newline at end of file diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 0bf99f7a5..0e9e3a360 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -349,13 +349,6 @@ class LBRYWallet(object): value['sources'][k] = sources[k] if value['sources'] == {}: return defer.fail("No source given") - for k in BASE_METADATA_FIELDS: - if k not in metadata: - return defer.fail("Missing required field '%s'" % k) - value[k] = metadata[k] - for k in metadata: - if k not in BASE_METADATA_FIELDS: - value[k] = metadata[k] if fee is not None: if "LBC" in fee: value['fee'] = {'LBC': {'amount': fee['LBC']['amount'], 'address': fee['LBC']['address']}} diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 8e2c98cc5..75aaa0096 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -32,6 +32,8 @@ from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerF from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory from lbrynet.core.Error import UnknownNameError, InsufficientFundsError +from lbrynet.core.LBRYFee import LBRYFee +from lbrynet.core.LBRYMetadata import Metadata from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier @@ -41,8 +43,7 @@ from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ - DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, \ - BASE_METADATA_FIELDS, OPTIONAL_METADATA_FIELDS, SOURCE_TYPES + DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES from lbrynet.conf import DEFAULT_TIMEOUT, WALLET_TYPES from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob from lbrynet.core.Session import LBRYSession @@ -169,6 +170,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.streams = {} self.known_dht_nodes = KNOWN_DHT_NODES self.first_run_after_update = False + self.last_traded_rate = None if os.name == "nt": from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle @@ -249,6 +251,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.session_settings['last_version'] = self.default_settings['last_version'] self.first_run_after_update = True log.info("First run after update") + log.info("lbrynet %s --> %s" % (self.session_settings['last_version']['lbrynet'], self.default_settings['last_version']['lbrynet'])) + log.info("lbryum %s --> %s" % (self.session_settings['last_version']['lbryum'], self.default_settings['last_version']['lbryum'])) f = open(self.daemon_conf, "w") f.write(json.dumps(self.session_settings)) @@ -337,6 +341,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker = LoopingCall(self._check_network_connection) self.version_checker = LoopingCall(self._check_remote_versions) self.connection_problem_checker = LoopingCall(self._check_connection_problems) + self.price_checker = LoopingCall(self._update_exchange) # self.lbrynet_connection_checker = LoopingCall(self._check_lbrynet_connection) self.sd_identifier = StreamDescriptorIdentifier() @@ -470,6 +475,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) self.connection_problem_checker.start(1) + self.price_checker.start(600) if host_ui: self.lbry_ui_manager.update_checker.start(1800, now=False) @@ -590,6 +596,13 @@ class LBRYDaemon(jsonrpc.JSONRPC): if not self.connected_to_internet: self.connection_problem = CONNECTION_PROBLEM_CODES[1] + def _update_exchange(self): + try: + r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'}) + self.last_traded_rate = float(json.loads(r.text)['result']['Last']) + except: + self.last_traded_rate = None + def _start_server(self): if self.peer_port is not None: @@ -701,6 +714,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.connection_problem_checker.stop() if self.lbry_ui_manager.update_checker.running: self.lbry_ui_manager.update_checker.stop() + if self.price_checker.running: + self.price_checker.stop() d = self._upload_log(log_type="close", exclude_previous=False if self.first_run else True) d.addCallback(lambda _: self._stop_server()) @@ -1868,9 +1883,9 @@ class LBRYDaemon(jsonrpc.JSONRPC): name = p['name'] bid = p['bid'] file_path = p['file_path'] - metadata = p['metadata'] + metadata = Metadata(p['metadata']) if 'fee' in p: - fee = p['fee'] + fee = LBRYFee.from_dict(p['fee']) else: fee = None diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index c72444c70..9cfbf07c1 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -11,6 +11,7 @@ from twisted.internet.task import LoopingCall from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob +from lbrynet.core.LBRYFee import LBRYFee from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME @@ -94,14 +95,9 @@ class GetStream(object): if 'description' in self.stream_info: self.description = self.stream_info['description'] if 'fee' in self.stream_info: - if 'LBC' in self.stream_info['fee']: - self.key_fee = float(self.stream_info['fee']['LBC']['amount']) - self.key_fee_address = self.stream_info['fee']['LBC']['address'] - else: - self.key_fee_address = None + self.fee = LBRYFee.from_dict(stream_info['fee']) else: - self.key_fee = None - self.key_fee_address = None + self.fee = None if self.key_fee > self.max_key_fee: log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name)) return defer.fail(None) diff --git a/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py b/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py index cd7ed02cb..92ea9f9f1 100644 --- a/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py +++ b/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py @@ -2,10 +2,13 @@ import json import logging.handlers import sys import os +import requests +from datetime import datetime from appdirs import user_data_dir from twisted.internet.task import LoopingCall from twisted.internet import reactor +from twisted.internet.threads import deferToThread if sys.platform != "darwin": @@ -39,11 +42,15 @@ class Autofetcher(object): def __init__(self, api): self._api = api self._checker = LoopingCall(self._check_for_new_claims) + self._price_checker = LoopingCall(self._update_price) self.best_block = None + self.last_price = None + self.price_updated = None def start(self): reactor.addSystemEventTrigger('before', 'shutdown', self.stop) self._checker.start(5) + self._price_checker.start(30) def stop(self): log.info("Stopping autofetcher") @@ -59,9 +66,24 @@ class Autofetcher(object): c = self._api.get_claims_for_tx({'txid': t}) if len(c): for i in c: + if 'fee' in json.loads(i['value']): + log.info("Downloading stream for claim txid: %s" % t) self._api.get({'name': t, 'stream_info': json.loads(i['value'])}) + def _update_price(self): + def _check_bittrex(): + try: + r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'}) + self.last_price = json.loads(r.text)['result']['Last'] + self.price_updated = datetime.now() + log.info("Updated exchange rate, last BTC-LBC trade: %f" % self.last_price) + except: + log.info("Failed to update exchange rate") + self.last_price = None + self.price_updated = datetime.now() + return deferToThread(_check_bittrex) + def run(api): fetcher = Autofetcher(api) From b65402db413fbe9dbc4560091c75245e9fff8a58 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 21 Jul 2016 16:47:47 -0400 Subject: [PATCH 03/73] debug line --- lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index 6a1fc7c9b..3fad1d126 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -68,7 +68,7 @@ class LBRYDaemonRequest(server.Request): from twisted.web.http import parse_qs if self.do_log: print '%f Request Received' % time.time() - print self.content + print self.content self.content.seek(0,0) self.args = {} From 6bfb33adb275e387ca18916dec56cae3fefdd1eb Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 24 Jul 2016 22:03:32 -0400 Subject: [PATCH 04/73] fix version comparison --- lbrynet/core/utils.py | 24 +++++++++++- lbrynet/lbrynet_daemon/LBRYDaemon.py | 55 +++++++++++++--------------- 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index 89b57fb0c..34efb4eb5 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -25,4 +25,26 @@ def is_valid_blobhash(blobhash): for l in blobhash: if l not in "0123456789abcdef": return False - return True \ No newline at end of file + return True + + +def version_is_greater_than(version1, version2): + """ + handles differing numbers of subversions, ie 0.3.10 > 0.3.9.9 + """ + + v1, v2 = version1.split("."), version2.split(".") + r = True + if len(v2) > len(v1): + for j in range(len(v2) - len(v1)): + v1.append("0") + elif len(v2) < len(v1): + for j in range(len(v1) - len(v2)): + v2.append("0") + for c in range(len(v1)): + if int(v2[c]) > int(v1[c]): + r = False + break + elif c == len(v1) - 1 and int(v1[c]) == int(v2[c]): + r = False + return r \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 09f8f0db4..38c6b861a 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -5,6 +5,8 @@ import os import subprocess import sys import random + +import pkg_resources import simplejson as json import binascii import logging.handlers @@ -32,7 +34,6 @@ from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerF from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory from lbrynet.core.Error import UnknownNameError, InsufficientFundsError -from lbrynet.core.LBRYFee import LBRYFee from lbrynet.core.LBRYMetadata import Metadata from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory @@ -40,7 +41,7 @@ from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifi from lbrynet.lbrynet_daemon.LBRYUIManager import LBRYUIManager from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher -from lbrynet.core.utils import generate_id +from lbrynet.core.utils import generate_id, version_is_greater_than from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES @@ -161,8 +162,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.connected_to_internet = True self.connection_problem = None self.query_handlers = {} - self.git_lbrynet_version = None - self.git_lbryum_version = None + self.pip_lbrynet_version = None + self.pip_lbryum_version = None self.ui_version = None self.ip = None # TODO: this is confusing to set here, and then to be reset below. @@ -576,39 +577,33 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _check_remote_versions(self): def _get_lbryum_version(): try: - r = urlopen("https://raw.githubusercontent.com/lbryio/lbryum/master/lib/version.py").read().split('\n') - version = next(line.split("=")[1].split("#")[0].replace(" ", "") - for line in r if "LBRYUM_VERSION" in line) - version = version.replace("'", "") - log.info("remote lbryum " + str(version) + " > local lbryum " + str(lbryum_version) + " = " + str( - version > lbryum_version)) - self.git_lbryum_version = version + r = pkg_resources.get_distribution("lbryum").version + log.info("Local lbryum: %s" % lbryum_version) + log.info("Available lbryum: %s" % r) + self.pip_lbryum_version = r return defer.succeed(None) except: log.info("Failed to get lbryum version from git") - self.git_lbryum_version = None + self.pip_lbryum_version = None return defer.fail(None) def _get_lbrynet_version(): try: - r = urlopen("https://raw.githubusercontent.com/lbryio/lbry/master/lbrynet/__init__.py").read().split('\n') - vs = next(i for i in r if '__version__ =' in i).split("=")[1].replace(" ", "") - vt = tuple(int(x) for x in vs[1:-1].split('.')) - vr = ".".join([str(x) for x in vt]) - log.info("remote lbrynet " + str(vr) + " > local lbrynet " + str(lbrynet_version) + " = " + str( - vr > lbrynet_version)) - self.git_lbrynet_version = vr + r = pkg_resources.get_distribution("lbrynet").version + log.info("Local lbrynet: %s" % lbrynet_version) + log.info("Available lbrynet: %s" % r) + self.pip_lbrynet_version = r return defer.succeed(None) except: log.info("Failed to get lbrynet version from git") - self.git_lbrynet_version = None + self.pip_lbrynet_version = None return defer.fail(None) d = _get_lbrynet_version() d.addCallback(lambda _: _get_lbryum_version()) def _check_connection_problems(self): - if not self.git_lbrynet_version or not self.git_lbryum_version: + if not self.pip_lbrynet_version or not self.pip_lbryum_version: self.connection_problem = CONNECTION_PROBLEM_CODES[0] elif self.startup_status[0] == 'loading_wallet': @@ -1091,7 +1086,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _disp_file(f): file_path = os.path.join(self.download_directory, f.file_name) - log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.sd_hash) + " --> " + file_path) + log.info("Already downloaded: " + str(f.sd_hash) + " --> " + file_path) return f def _get_stream(stream_info): @@ -1162,18 +1157,18 @@ class LBRYDaemon(jsonrpc.JSONRPC): if not force_refresh: if name in self.name_cache.keys(): if (self._get_long_count_timestamp() - self.name_cache[name]['timestamp']) < self.cache_time: - log.info("[" + str(datetime.now()) + "] Returning cached stream info for lbry://" + name) + log.info("Returning cached stream info for lbry://" + name) d = defer.succeed(self.name_cache[name]['claim_metadata']) else: - log.info("[" + str(datetime.now()) + "] Refreshing stream info for lbry://" + name) + log.info("Refreshing stream info for lbry://" + name) d = self.session.wallet.get_stream_info_for_name(name) d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) else: - log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) + log.info("Resolving stream info for lbry://" + name) d = self.session.wallet.get_stream_info_for_name(name) d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) else: - log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) + log.info("Resolving stream info for lbry://" + name) d = self.session.wallet.get_stream_info_for_name(name) d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) @@ -1506,10 +1501,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'lbrynet_version': lbrynet_version, 'lbryum_version': lbryum_version, 'ui_version': self.ui_version, - 'remote_lbrynet': self.git_lbrynet_version, - 'remote_lbryum': self.git_lbryum_version, - 'lbrynet_update_available': lbrynet_version < self.git_lbrynet_version, - 'lbryum_update_available': lbryum_version < self.git_lbryum_version + 'remote_lbrynet': self.pip_lbrynet_version, + 'remote_lbryum': self.pip_lbryum_version, + 'lbrynet_update_available': version_is_greater_than(self.pip_lbrynet_version, lbrynet_version), + 'lbryum_update_available': version_is_greater_than(self.pip_lbryum_version, lbryum_version), } log.info("Get version info: " + json.dumps(msg)) From 2e2d309d38f29ed2aa11cc51a247c6b954dd2039 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 01:40:26 -0400 Subject: [PATCH 05/73] BTC/USD denominated fees, metadata revisions --- lbrynet/conf.py | 2 + lbrynet/core/LBRYFee.py | 39 --------------- lbrynet/core/LBRYMetadata.py | 64 ++++++++++++++++++++++-- lbrynet/core/LBRYWallet.py | 34 ++++--------- lbrynet/lbrynet_daemon/LBRYDaemon.py | 26 ++++++---- lbrynet/lbrynet_daemon/LBRYDownloader.py | 35 +++++-------- lbrynet/lbrynet_daemon/LBRYPublisher.py | 26 +++------- 7 files changed, 112 insertions(+), 114 deletions(-) delete mode 100644 lbrynet/core/LBRYFee.py diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 736381c4a..fd5e24f54 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -51,4 +51,6 @@ SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] CURRENCIES = [ {'BTC': {'type': 'crypto'}}, {'LBC': {'type': 'crypto'}}, + {'USD': {'type': 'fiat'}}, + ] diff --git a/lbrynet/core/LBRYFee.py b/lbrynet/core/LBRYFee.py deleted file mode 100644 index 09b7dbd39..000000000 --- a/lbrynet/core/LBRYFee.py +++ /dev/null @@ -1,39 +0,0 @@ -import requests -import json - -from lbrynet.conf import CURRENCIES - - -class LBRYFee(object): - def __init__(self, currency, amount, address=None): - assert currency in [c.keys()[0] for c in CURRENCIES], "Unsupported currency: %s" % str(currency) - self.address = address - self.currency_symbol = currency - self.currency = [c for c in CURRENCIES if self.currency_symbol in c][0] - if not isinstance(amount, float): - self.amount = float(amount) - else: - self.amount = amount - - def convert_to(self, to_currency, rate_dict={}): - if to_currency is self.currency_symbol: - return self.as_dict() - if self.currency[self.currency_symbol]['type'] is 'fiat': - raise NotImplemented - else: - if to_currency not in rate_dict: - params = {'market': '%s-%s' % (self.currency_symbol, to_currency)} - r = requests.get("https://bittrex.com/api/v1.1/public/getticker", params) - last = json.loads(r.text)['result']['Last'] - converted = self.amount / float(last) - else: - converted = self.amount / float(rate_dict[to_currency]['last']) - - return LBRYFee(to_currency, converted, self.address).as_dict() - - def as_dict(self): - return {self.currency_symbol: {'amount': self.amount, 'address': self.address}} - - def from_dict(self, fee_dict): - s = fee_dict.keys()[0] - return LBRYFee(s, fee_dict[s]['amount'], fee_dict[s]['address']) \ No newline at end of file diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 565a10785..efbb334de 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -1,13 +1,63 @@ +import requests import json -BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type'] +from googlefinance import getQuotes +from lbrynet.conf import CURRENCIES + +SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] + +BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] #v0.0.1 metadata METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}} #v0.0.2 metadata additions -METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw'], 'optional': []} +METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['licence_url']} + +CURRENT_METADATA_VERSION = '0.0.2' + +class LBRYFee(object): + def __init__(self, currency, amount, address=None): + assert currency in [c.keys()[0] for c in CURRENCIES], "Unsupported currency: %s" % str(currency) + self.address = address + self.currency_symbol = currency + self.currency = next(c for c in CURRENCIES if self.currency_symbol in c) + if not isinstance(amount, float): + self.amount = float(amount) + else: + self.amount = amount + + def __call__(self): + return {self.currency_symbol: {'amount': self.amount, 'address': self.address}} + + def convert(self, amount_only=False): + if self.currency_symbol == "LBC": + r = round(float(self.amount), 5) + elif self.currency_symbol == "BTC": + r = round(float(self.BTC_to_LBC(self.amount)), 5) + elif self.currency_symbol == "USD": + r = round(float(self.BTC_to_LBC(self.USD_to_BTC(self.amount))), 5) + + if not amount_only: + return {'LBC': {'amount': r, 'address': self.address}} + else: + return r + + def USD_to_BTC(self, usd): + r = float(getQuotes('CURRENCY:%sBTC' % self.currency_symbol)[0]['LastTradePrice']) * float(usd) + return r + + def BTC_to_LBC(self, btc): + r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'}) + last = json.loads(r.text)['result']['Last'] + converted = float(btc) / float(last) + return converted + + +def fee_from_dict(fee_dict): + s = fee_dict.keys()[0] + return LBRYFee(s, fee_dict[s]['amount'], fee_dict[s]['address']) class Metadata(dict): @@ -15,6 +65,14 @@ class Metadata(dict): dict.__init__(self) self.metaversion = None m = metadata.copy() + + if 'fee' in metadata: + assert fee_from_dict(metadata['fee']) + + assert "sources" in metadata, "No sources given" + for source in metadata['sources']: + assert source in SOURCE_TYPES, "Unknown source type" + for version in METADATA_REVISIONS: for k in METADATA_REVISIONS[version]['required']: assert k in metadata, "Missing required metadata field: %s" % k @@ -25,4 +83,4 @@ class Metadata(dict): if not len(m): self.metaversion = version break - assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys()) \ No newline at end of file + assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys()) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index e5f7f1889..278396186 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -4,7 +4,8 @@ from lbrynet.core.client.ClientRequest import ClientRequest from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError from lbrynet.core.Error import InsufficientFundsError from lbrynet.core.sqlite_helpers import rerun_if_locked -from lbrynet.conf import BASE_METADATA_FIELDS, SOURCE_TYPES, OPTIONAL_METADATA_FIELDS +from lbrynet.conf import SOURCE_TYPES +from lbrynet.core.LBRYMetadata import Metadata from lbryum import SimpleConfig, Network from lbryum.lbrycrd import COIN, TYPE_ADDRESS @@ -316,7 +317,6 @@ class LBRYWallet(object): return d def _get_stream_info_from_value(self, result, name): - r_dict = {} if 'value' in result: value = result['value'] @@ -324,16 +324,11 @@ class LBRYWallet(object): value_dict = json.loads(value) except (ValueError, TypeError): return Failure(InvalidStreamInfoError(name)) - r_dict['sources'] = value_dict['sources'] - for field in BASE_METADATA_FIELDS: - r_dict[field] = value_dict[field] - for field in value_dict: - if field in OPTIONAL_METADATA_FIELDS: - r_dict[field] = value_dict[field] - + m = Metadata(value_dict) if 'txid' in result: - d = self._save_name_metadata(name, r_dict['sources']['lbry_sd_hash'], str(result['txid'])) - d.addCallback(lambda _: r_dict) + d = self._save_name_metadata(name, m['sources']['lbry_sd_hash'], str(result['txid'])) + d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, m.metaversion))) + d.addCallback(lambda _: m) return d elif 'error' in result: log.warning("Got an error looking up a name: %s", result['error']) @@ -342,21 +337,14 @@ class LBRYWallet(object): log.warning("Got an error looking up a name: %s", json.dumps(result)) return Failure(UnknownNameError(name)) - def claim_name(self, name, bid, sources, metadata, fee=None): - value = {'sources': {}} - for k in SOURCE_TYPES: - if k in sources: - value['sources'][k] = sources[k] - if value['sources'] == {}: - return defer.fail("No source given") - if fee is not None: - if "LBC" in fee: - value['fee'] = {'LBC': {'amount': fee['LBC']['amount'], 'address': fee['LBC']['address']}} + def claim_name(self, name, bid, m): - d = self._send_name_claim(name, json.dumps(value), bid) + metadata = Metadata(m) + + d = self._send_name_claim(name, json.dumps(metadata), bid) def _save_metadata(txid): - d = self._save_name_metadata(name, value['sources']['lbry_sd_hash'], txid) + d = self._save_name_metadata(name, metadata['sources']['lbry_sd_hash'], txid) d.addCallback(lambda _: txid) return d diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 38c6b861a..fb510a0cc 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -54,8 +54,6 @@ from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager # from lbryum import LOG_PATH as lbryum_log -log = logging.getLogger(__name__) - if sys.platform != "darwin": log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") @@ -67,13 +65,12 @@ if not os.path.isdir(log_dir): lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) -log = logging.getLogger(__name__) - # TODO: configuring a logger on module import drastically reduces the # amount of control the caller of this code has over logging # # Better would be to configure all logging at runtime. handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) +log = logging.getLogger(__name__) log.addHandler(handler) log.setLevel(logging.INFO) @@ -1908,19 +1905,30 @@ class LBRYDaemon(jsonrpc.JSONRPC): Returns: Claim txid """ - # start(self, name, file_path, bid, metadata, fee=None, sources=None): + name = p['name'] bid = p['bid'] file_path = p['file_path'] - metadata = Metadata(p['metadata']) + metadata = p['metadata'] + + def _set_address(address): + metadata['fee']['address'] = address + return defer.succeed(None) + if 'fee' in p: - fee = LBRYFee.from_dict(p['fee']) + metadata['fee'] = p['fee'] + if 'address' not in metadata['fee']: + d = self.session.wallet.get_new_address() + d.addCallback(_set_address) + else: + d = defer.succeed(None) else: - fee = None + d = defer.succeed(None) + pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) - d = pub.start(name, file_path, bid, metadata, fee) + d.addCallback(lambda _: pub.start(name, file_path, bid, metadata)) d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index eddffa0b5..7ae0ac13f 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -11,7 +11,7 @@ from twisted.internet.task import LoopingCall from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob -from lbrynet.core.LBRYFee import LBRYFee +from lbrynet.core.LBRYMetadata import Metadata, fee_from_dict from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME @@ -48,8 +48,7 @@ class GetStream(object): self.wallet = wallet self.resolved_name = None self.description = None - self.key_fee = None - self.key_fee_address = None + self.fee = None self.data_rate = data_rate self.name = None self.file_name = file_name @@ -59,7 +58,7 @@ class GetStream(object): self.sd_identifier = sd_identifier self.stream_hash = None self.max_key_fee = max_key_fee - self.stream_info = None + self.metadata = None self.stream_info_manager = None self.d = defer.Deferred(None) self.timeout = timeout @@ -87,22 +86,14 @@ class GetStream(object): def start(self, stream_info, name): self.resolved_name = name - self.stream_info = stream_info - if 'sources' in self.stream_info: - self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] - else: - raise InvalidStreamInfoError(self.stream_info) - if 'description' in self.stream_info: - self.description = self.stream_info['description'] - if 'fee' in self.stream_info: - self.fee = LBRYFee.from_dict(stream_info['fee']) - else: - self.fee = None - if self.key_fee > self.max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name)) - return defer.fail(None) - else: - pass + self.metadata = stream_info + self.stream_hash = self.metadata['sources']['lbry_sd_hash'] + + if 'fee' in self.metadata: + fee = fee_from_dict(self.metadata['fee']) + if fee.convert(amount_only=True) > self.max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name)) + return defer.fail(None) def _cause_timeout(): self.timeout_counter = self.timeout * 2 @@ -132,8 +123,8 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): - if self.key_fee is not None and self.key_fee_address is not None: - reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee) + if self.fee is not None: + reserved_points = self.wallet.reserve_points(self.fee.address, self.fee.convert(amount_only=True)) if reserved_points is None: return defer.fail(InsufficientFundsError()) log.info("Key fee: %f --> %s" % (self.key_fee, self.key_fee_address)) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index bd5c50e4b..10f787209 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -10,6 +10,7 @@ from lbrynet.core.Error import InsufficientFundsError from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.core.PaymentRateManager import PaymentRateManager +from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader from lbrynet.conf import LOG_FILE_NAME from twisted.internet import threads, defer @@ -42,10 +43,9 @@ class Publisher(object): self.verified = False self.lbry_file = None self.txid = None - self.sources = {} - self.fee = None + self.metadata = {} - def start(self, name, file_path, bid, metadata, fee=None, sources={}): + def start(self, name, file_path, bid, metadata): def _show_result(): log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) @@ -54,7 +54,6 @@ class Publisher(object): self.publish_name = name self.file_path = file_path self.bid_amount = bid - self.fee = fee self.metadata = metadata d = self._check_file_path(self.file_path) @@ -75,16 +74,6 @@ class Publisher(object): return True return threads.deferToThread(check_file_threaded) - def _get_new_address(self): - d = self.wallet.get_new_address() - - def set_address(address): - self.key_fee_address = address - return True - - d.addCallback(set_address) - return d - def set_status(self, lbry_file_downloader): self.lbry_file = lbry_file_downloader d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED) @@ -102,7 +91,9 @@ class Publisher(object): self.lbry_file.stream_hash) def set_sd_hash(sd_hash): - self.sources['lbry_sd_hash'] = sd_hash + if 'sources' not in self.metadata: + self.metadata['sources'] = {} + self.metadata['sources']['lbry_sd_hash'] = sd_hash d.addCallback(set_sd_hash) return d @@ -110,11 +101,10 @@ class Publisher(object): def _claim_name(self): self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name))[0] + self.metadata['ver'] = CURRENT_METADATA_VERSION d = self.wallet.claim_name(self.publish_name, self.bid_amount, - self.sources, - self.metadata, - fee=self.fee) + Metadata(self.metadata)) def set_tx_hash(txid): self.txid = txid From 75ef652cb300ab7ad3a1bf6f638fd8b2cfc91e5d Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 17:08:56 -0400 Subject: [PATCH 06/73] add googlefinance to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 60f800df2..847b69ee1 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ console_scripts = ['lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUp requires = ['pycrypto', 'twisted', 'miniupnpc', 'yapsy', 'seccure', 'python-bitcoinrpc==0.1', 'txJSON-RPC', 'requests>=2.4.2', 'unqlite==0.2.0', - 'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58'] + 'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58', 'googlefinance'] setup(name='lbrynet', description='A decentralized media library and marketplace', From c4a78a149bd3b3f1f021e5d32c6766ce89424a41 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 21:45:42 -0400 Subject: [PATCH 07/73] LBRYFee things -move exchange rate updates to wallet, do them every half hour -add convention checker for fees -pay BTC or USD denominated key fees in converted LBC amount --- lbrynet/core/LBRYMetadata.py | 119 +++++++++++++++-------- lbrynet/core/LBRYWallet.py | 90 +++++++++++++---- lbrynet/lbrynet_daemon/LBRYDownloader.py | 15 +-- 3 files changed, 159 insertions(+), 65 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index efbb334de..4090a2f2a 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -1,63 +1,105 @@ import requests import json +import time from googlefinance import getQuotes from lbrynet.conf import CURRENCIES +import logging + +log = logging.getLogger(__name__) + +BITTREX_FEE = 0.0025 SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] - BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] #v0.0.1 metadata METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}} - #v0.0.2 metadata additions METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['licence_url']} - CURRENT_METADATA_VERSION = '0.0.2' + +#v0.0.1 fee +FEE_REVISIONS = {'0.0.1': {'required': ['amount', 'address'], 'optional': []}} +CURRENT_FEE_REVISION = '0.0.1' + + class LBRYFee(object): - def __init__(self, currency, amount, address=None): - assert currency in [c.keys()[0] for c in CURRENCIES], "Unsupported currency: %s" % str(currency) - self.address = address - self.currency_symbol = currency - self.currency = next(c for c in CURRENCIES if self.currency_symbol in c) - if not isinstance(amount, float): - self.amount = float(amount) - else: - self.amount = amount + def __init__(self, fee_dict, rate_dict): + fee = LBRYFeeFormat(fee_dict) - def __call__(self): - return {self.currency_symbol: {'amount': self.amount, 'address': self.address}} + for currency in fee: + self.address = fee[currency]['address'] + if not isinstance(fee[currency]['amount'], float): + self.amount = float(fee[currency]['amount']) + else: + self.amount = fee[currency]['amount'] + self.currency_symbol = currency - def convert(self, amount_only=False): + assert 'BTCLBC' in rate_dict and 'USDBTC' in rate_dict + for fx in rate_dict: + assert int(time.time()) - int(rate_dict[fx]['ts']) < 3600, "%s quote is out of date" % fx + self._USDBTC = {'spot': rate_dict['USDBTC']['spot'], 'ts': rate_dict['USDBTC']['ts']} + self._BTCLBC = {'spot': rate_dict['BTCLBC']['spot'], 'ts': rate_dict['BTCLBC']['ts']} + + def to_lbc(self): + r = None if self.currency_symbol == "LBC": r = round(float(self.amount), 5) elif self.currency_symbol == "BTC": - r = round(float(self.BTC_to_LBC(self.amount)), 5) + r = round(float(self._btc_to_lbc(self.amount)), 5) elif self.currency_symbol == "USD": - r = round(float(self.BTC_to_LBC(self.USD_to_BTC(self.amount))), 5) - - if not amount_only: - return {'LBC': {'amount': r, 'address': self.address}} - else: - return r - - def USD_to_BTC(self, usd): - r = float(getQuotes('CURRENCY:%sBTC' % self.currency_symbol)[0]['LastTradePrice']) * float(usd) + r = round(float(self._btc_to_lbc(self._usd_to_btc(self.amount))), 5) + assert r is not None return r - def BTC_to_LBC(self, btc): - r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'}) - last = json.loads(r.text)['result']['Last'] - converted = float(btc) / float(last) - return converted + def to_usd(self): + r = None + if self.currency_symbol == "USD": + r = round(float(self.amount), 5) + elif self.currency_symbol == "BTC": + r = round(float(self._btc_to_usd(self.amount)), 5) + elif self.currency_symbol == "LBC": + r = round(float(self._btc_to_usd(self._lbc_to_btc(self.amount))), 5) + assert r is not None + return r + + def _usd_to_btc(self, usd): + return self._USDBTC['spot'] * float(usd) + + def _btc_to_usd(self, btc): + return float(btc) / self._USDBTC['spot'] + + def _btc_to_lbc(self, btc): + return float(btc) / self._BTCLBC['spot'] / (1.0 - BITTREX_FEE) + + def _lbc_to_btc(self, lbc): + return self._BTCLBC['spot'] * float(lbc) -def fee_from_dict(fee_dict): - s = fee_dict.keys()[0] - return LBRYFee(s, fee_dict[s]['amount'], fee_dict[s]['address']) +class LBRYFeeFormat(dict): + def __init__(self, fee_dict): + dict.__init__(self) + self.fee_version = None + f = fee_dict.copy() + assert len(fee_dict) == 1 + for currency in fee_dict: + assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) + self.update({currency: {}}) + + for version in FEE_REVISIONS: + for k in FEE_REVISIONS[version]['required']: + assert k in fee_dict, "Missing required fee field: %s" % k + self[currency].update({k: f.pop(k)}) + for k in FEE_REVISIONS[version]['optional']: + if k in fee_dict: + self[currency].update({k: f.pop(k)}) + if not len(f): + self.fee_version = version + break + assert f == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) class Metadata(dict): @@ -66,9 +108,6 @@ class Metadata(dict): self.metaversion = None m = metadata.copy() - if 'fee' in metadata: - assert fee_from_dict(metadata['fee']) - assert "sources" in metadata, "No sources given" for source in metadata['sources']: assert source in SOURCE_TYPES, "Unknown source type" @@ -78,9 +117,13 @@ class Metadata(dict): assert k in metadata, "Missing required metadata field: %s" % k self.update({k: m.pop(k)}) for k in METADATA_REVISIONS[version]['optional']: - if k in metadata: + if k == 'fee': + pass + elif k in metadata: self.update({k: m.pop(k)}) - if not len(m): + if not len(m) or m.keys() == ['fee']: self.metaversion = version break + if 'fee' in m: + self.update({'fee': LBRYFeeFormat(m.pop('fee'))}) assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys()) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 278396186..e09bd020d 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -1,17 +1,12 @@ import sys -from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet -from lbrynet.core.client.ClientRequest import ClientRequest -from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError -from lbrynet.core.Error import InsufficientFundsError -from lbrynet.core.sqlite_helpers import rerun_if_locked -from lbrynet.conf import SOURCE_TYPES -from lbrynet.core.LBRYMetadata import Metadata - -from lbryum import SimpleConfig, Network -from lbryum.lbrycrd import COIN, TYPE_ADDRESS -from lbryum.wallet import WalletStorage, Wallet -from lbryum.commands import known_commands, Commands -from lbryum.transaction import Transaction +import datetime +import logging +import json +import subprocess +import socket +import time +import os +import requests from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from twisted.internet import threads, reactor, defer, task @@ -20,15 +15,24 @@ from twisted.enterprise import adbapi from collections import defaultdict, deque from zope.interface import implements from decimal import Decimal -import datetime -import logging -import json -import subprocess -import socket -import time -import os +from googlefinance import getQuotes + +from lbryum import SimpleConfig, Network +from lbryum.lbrycrd import COIN, TYPE_ADDRESS +from lbryum.wallet import WalletStorage, Wallet +from lbryum.commands import known_commands, Commands +from lbryum.transaction import Transaction + +from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet +from lbrynet.core.client.ClientRequest import ClientRequest +from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError +from lbrynet.core.Error import InsufficientFundsError +from lbrynet.core.sqlite_helpers import rerun_if_locked +from lbrynet.conf import SOURCE_TYPES +from lbrynet.core.LBRYMetadata import Metadata log = logging.getLogger(__name__) +log.setLevel(logging.INFO) alert = logging.getLogger("lbryalert." + __name__) @@ -80,6 +84,50 @@ class LBRYWallet(object): self._batch_count = 20 self._first_run = self._FIRST_RUN_UNKNOWN + self._USDBTC = None + self._BTCLBC = None + self._exchange_rate_updater = task.LoopingCall(self._update_exchange_rates) + + def _usd_to_btc(self): + if self._USDBTC is not None: + if int(time.time()) - int(self._USDBTC['ts']) < 600: + log.info("USDBTC quote is new enough") + return defer.succeed({}) + + log.info("Getting new USDBTC quote") + x = float(getQuotes('CURRENCY:USDBTC')[0]['LastTradePrice']) + return defer.succeed({'USDBTC': {'spot': x, 'ts': int(time.time())}}) + + def _btc_to_lbc(self): + if self._BTCLBC is not None: + if int(time.time()) - int(self._BTCLBC['ts']) < 600: + log.info("BTCLBC quote is new enough") + return defer.succeed({}) + + log.info("Getting new BTCLBC quote") + r = requests.get("https://bittrex.com/api/v1.1/public/getmarkethistory", {'market': 'BTC-LBC', 'count': 50}) + trades = json.loads(r.text)['result'] + vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades]) + x = (1.0 / float(vwap)) / 0.99975 + + return defer.succeed({'BTCLBC': {'spot': x, 'ts': int(time.time())}}) + + def _set_exchange_rates(self, rates): + if 'USDBTC' in rates: + assert int(time.time()) - int(rates['USDBTC']['ts']) < 3600, "new USDBTC quote is too old" + self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']} + log.info("Updated USDBTC rate: %s" % json.dumps(self._USDBTC)) + if 'BTCLBC' in rates: + assert int(time.time()) - int(rates['BTCLBC']['ts']) < 3600, "new BTCLBC quote is too old" + self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']} + log.info("Updated BTCLBC rate: %s" % json.dumps(self._BTCLBC)) + + def _update_exchange_rates(self): + d = self._usd_to_btc() + d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) + d.addCallback(lambda _: self._btc_to_lbc()) + d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) + def start(self): def start_manage(): @@ -87,6 +135,8 @@ class LBRYWallet(object): self.manage() return True + self._exchange_rate_updater.start(1800) + d = self._open_db() d.addCallback(lambda _: self._start()) d.addCallback(lambda _: start_manage()) diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 7ae0ac13f..7563f018b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -11,7 +11,7 @@ from twisted.internet.task import LoopingCall from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob -from lbrynet.core.LBRYMetadata import Metadata, fee_from_dict +from lbrynet.core.LBRYMetadata import Metadata, LBRYFee from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME @@ -90,9 +90,9 @@ class GetStream(object): self.stream_hash = self.metadata['sources']['lbry_sd_hash'] if 'fee' in self.metadata: - fee = fee_from_dict(self.metadata['fee']) - if fee.convert(amount_only=True) > self.max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name)) + self.fee = LBRYFee(self.metadata['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + if self.fee.to_lbc() > self.max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) return defer.fail(None) def _cause_timeout(): @@ -124,11 +124,12 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): if self.fee is not None: - reserved_points = self.wallet.reserve_points(self.fee.address, self.fee.convert(amount_only=True)) + x = self.fee.to_lbc() + reserved_points = self.wallet.reserve_points(self.fee.address, x) if reserved_points is None: return defer.fail(InsufficientFundsError()) - log.info("Key fee: %f --> %s" % (self.key_fee, self.key_fee_address)) - return self.wallet.send_points_to_address(reserved_points, self.key_fee) + log.info("Key fee: %f --> %s" % (x, self.fee.address)) + return self.wallet.send_points_to_address(reserved_points, self.fee.address) return defer.succeed(None) d = _pay_key_fee() From a53e911a0c1141953112bb66be8ac19c901c0867 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 21:48:39 -0400 Subject: [PATCH 08/73] stop looping call when stop() is called --- lbrynet/core/LBRYWallet.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index e09bd020d..aa9df4127 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -149,6 +149,10 @@ class LBRYWallet(object): def stop(self): self.stopped = True + + if self._exchange_rate_updater.running: + self._exchange_rate_updater.stop() + # If self.next_manage_call is None, then manage is currently running or else # start has not been called, so set stopped and do nothing else. if self.next_manage_call is not None: From a5374cb876c1269b019c45d0107e4b6d1b777b87 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 22:17:31 -0400 Subject: [PATCH 09/73] fix currencies and LBRYFeeFormat --- lbrynet/conf.py | 11 +++++------ lbrynet/core/LBRYMetadata.py | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index fd5e24f54..cb626d717 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -48,9 +48,8 @@ DEFAULT_CACHE_TIME = 3600 DEFAULT_UI_BRANCH = "master" SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] -CURRENCIES = [ - {'BTC': {'type': 'crypto'}}, - {'LBC': {'type': 'crypto'}}, - {'USD': {'type': 'fiat'}}, - -] +CURRENCIES = { + 'BTC': {'type': 'crypto'}, + 'LBC': {'type': 'crypto'}, + 'USD': {'type': 'fiat'}, + } diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 4090a2f2a..e00833e20 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -88,18 +88,17 @@ class LBRYFeeFormat(dict): for currency in fee_dict: assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) self.update({currency: {}}) - for version in FEE_REVISIONS: for k in FEE_REVISIONS[version]['required']: - assert k in fee_dict, "Missing required fee field: %s" % k - self[currency].update({k: f.pop(k)}) + assert k in fee_dict[currency], "Missing required fee field: %s" % k + self[currency].update({k: f[currency].pop(k)}) for k in FEE_REVISIONS[version]['optional']: - if k in fee_dict: - self[currency].update({k: f.pop(k)}) + if k in fee_dict[currency]: + self[currency].update({k: f[currency].pop(k)}) if not len(f): self.fee_version = version break - assert f == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) + assert f[currency] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) class Metadata(dict): From d469ef7bf2e6be79c15079069942d0c478915ec7 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 22:21:38 -0400 Subject: [PATCH 10/73] add googlefinance to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 780abee9d..800ecf2ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ unqlite==0.2.0 wsgiref==0.1.2 zope.interface==4.1.3 base58==0.2.2 +googlefinance==0.7 \ No newline at end of file From 49c26a76b49325bffb5a15f434dc82826dbf8337 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 22:28:01 -0400 Subject: [PATCH 11/73] remove fee checking from autofetcher script --- .../daemon_scripts/Autofetcher.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py b/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py index 92ea9f9f1..cd7ed02cb 100644 --- a/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py +++ b/lbrynet/lbrynet_daemon/daemon_scripts/Autofetcher.py @@ -2,13 +2,10 @@ import json import logging.handlers import sys import os -import requests -from datetime import datetime from appdirs import user_data_dir from twisted.internet.task import LoopingCall from twisted.internet import reactor -from twisted.internet.threads import deferToThread if sys.platform != "darwin": @@ -42,15 +39,11 @@ class Autofetcher(object): def __init__(self, api): self._api = api self._checker = LoopingCall(self._check_for_new_claims) - self._price_checker = LoopingCall(self._update_price) self.best_block = None - self.last_price = None - self.price_updated = None def start(self): reactor.addSystemEventTrigger('before', 'shutdown', self.stop) self._checker.start(5) - self._price_checker.start(30) def stop(self): log.info("Stopping autofetcher") @@ -66,24 +59,9 @@ class Autofetcher(object): c = self._api.get_claims_for_tx({'txid': t}) if len(c): for i in c: - if 'fee' in json.loads(i['value']): - log.info("Downloading stream for claim txid: %s" % t) self._api.get({'name': t, 'stream_info': json.loads(i['value'])}) - def _update_price(self): - def _check_bittrex(): - try: - r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'}) - self.last_price = json.loads(r.text)['result']['Last'] - self.price_updated = datetime.now() - log.info("Updated exchange rate, last BTC-LBC trade: %f" % self.last_price) - except: - log.info("Failed to update exchange rate") - self.last_price = None - self.price_updated = datetime.now() - return deferToThread(_check_bittrex) - def run(api): fetcher = Autofetcher(api) From 102436274fcb353e3b64fa30138b1f0c68f6f64b Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 22:46:04 -0400 Subject: [PATCH 12/73] USD denominated max_key_fee in settings --- lbrynet/core/Error.py | 4 +++ lbrynet/lbrynet_daemon/LBRYDownloader.py | 33 ++++++++++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index b2c4c3360..631c5685c 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -22,6 +22,10 @@ class ConnectionClosedBeforeResponseError(Exception): pass +class KeyFeeAboveMaxAllowed(Exception): + pass + + class UnknownNameError(Exception): def __init__(self, name): self.name = name diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 7563f018b..852b0f367 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -8,7 +8,7 @@ from datetime import datetime from twisted.internet import defer from twisted.internet.task import LoopingCall -from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError +from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob from lbrynet.core.LBRYMetadata import Metadata, LBRYFee @@ -124,17 +124,34 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): if self.fee is not None: - x = self.fee.to_lbc() - reserved_points = self.wallet.reserve_points(self.fee.address, x) - if reserved_points is None: - return defer.fail(InsufficientFundsError()) - log.info("Key fee: %f --> %s" % (x, self.fee.address)) - return self.wallet.send_points_to_address(reserved_points, self.fee.address) + if isinstance(self.max_key_fee, int): + fee_lbc = self.fee.to_lbc() + if fee_lbc > self.max_key_fee: + return defer.fail(KeyFeeAboveMaxAllowed()) + reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) + if reserved_points is None: + return defer.fail(InsufficientFundsError()) + log.info("Key fee: %f --> %s" % (fee_lbc, self.fee.address)) + return self.wallet.send_points_to_address(reserved_points, self.fee.address) + else: + assert "USD" in self.max_key_fee + max_fee = LBRYFee(self.max_key_fee, {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + fee_lbc = self.fee.to_lbc() + if fee_lbc > max_fee.to_lbc(): + return defer.fail(KeyFeeAboveMaxAllowed()) + reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) + if reserved_points is None: + return defer.fail(InsufficientFundsError()) + log.info("Key fee: %f --> %s" % (fee_lbc, self.fee.address)) + return self.wallet.send_points_to_address(reserved_points, self.fee.address) + + return defer.succeed(None) - d = _pay_key_fee() self.downloader = downloader self.download_path = os.path.join(downloader.download_directory, downloader.file_name) + + d = _pay_key_fee() d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name)) d.addCallback(lambda _: self.downloader.start()) From 7e5af11c2de714474c31fad4b65185ba80aa9143 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 23:41:26 -0400 Subject: [PATCH 13/73] fix key fee payment problem --- lbrynet/core/LBRYMetadata.py | 4 +-- lbrynet/lbrynet_daemon/LBRYDownloader.py | 40 ++++++++++-------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index e00833e20..d5a4e6009 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -73,10 +73,10 @@ class LBRYFee(object): return float(btc) / self._USDBTC['spot'] def _btc_to_lbc(self, btc): - return float(btc) / self._BTCLBC['spot'] / (1.0 - BITTREX_FEE) + return float(btc) * self._BTCLBC['spot'] / (1.0 - BITTREX_FEE) def _lbc_to_btc(self, lbc): - return self._BTCLBC['spot'] * float(lbc) + return self._BTCLBC['spot'] / float(lbc) class LBRYFeeFormat(dict): diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 852b0f367..8d25aa56f 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -91,9 +91,15 @@ class GetStream(object): if 'fee' in self.metadata: self.fee = LBRYFee(self.metadata['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) - if self.fee.to_lbc() > self.max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) - return defer.fail(None) + if isinstance(self.max_key_fee, float): + if self.fee.to_lbc() > self.max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) + return defer.fail(KeyFeeAboveMaxAllowed()) + elif isinstance(self.max_key_fee, dict): + max_key = LBRYFee(self.max_key_fee, {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + if self.fee.to_lbc() > max_key.to_lbc(): + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), max_key.to_lbc(), self.resolved_name)) + return defer.fail(KeyFeeAboveMaxAllowed()) def _cause_timeout(): self.timeout_counter = self.timeout * 2 @@ -124,27 +130,13 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): if self.fee is not None: - if isinstance(self.max_key_fee, int): - fee_lbc = self.fee.to_lbc() - if fee_lbc > self.max_key_fee: - return defer.fail(KeyFeeAboveMaxAllowed()) - reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) - if reserved_points is None: - return defer.fail(InsufficientFundsError()) - log.info("Key fee: %f --> %s" % (fee_lbc, self.fee.address)) - return self.wallet.send_points_to_address(reserved_points, self.fee.address) - else: - assert "USD" in self.max_key_fee - max_fee = LBRYFee(self.max_key_fee, {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) - fee_lbc = self.fee.to_lbc() - if fee_lbc > max_fee.to_lbc(): - return defer.fail(KeyFeeAboveMaxAllowed()) - reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) - if reserved_points is None: - return defer.fail(InsufficientFundsError()) - log.info("Key fee: %f --> %s" % (fee_lbc, self.fee.address)) - return self.wallet.send_points_to_address(reserved_points, self.fee.address) - + fee_lbc = self.fee.to_lbc() + reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) + if reserved_points is None: + return defer.fail(InsufficientFundsError()) + log.info("Key fee: %f --> %s" % (fee_lbc, self.fee.address)) + d = self.wallet.send_points_to_address(reserved_points, self.fee.address) + return d return defer.succeed(None) From ca45f461ae7b11fddb678096067d022ae88ee910 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 23:42:03 -0400 Subject: [PATCH 14/73] spelling --- lbrynet/core/LBRYMetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index d5a4e6009..4e7f8ad39 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -17,7 +17,7 @@ OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] #v0.0.1 metadata METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}} #v0.0.2 metadata additions -METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['licence_url']} +METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']} CURRENT_METADATA_VERSION = '0.0.2' From 9e8827f015b54757127f7eed792885a1853d1d58 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 23:46:18 -0400 Subject: [PATCH 15/73] resolve name bug --- lbrynet/core/LBRYMetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 4e7f8ad39..133c34482 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -124,5 +124,5 @@ class Metadata(dict): self.metaversion = version break if 'fee' in m: - self.update({'fee': LBRYFeeFormat(m.pop('fee'))}) + self['fee'] = LBRYFeeFormat(m.pop('fee')) assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys()) From 491d431ea57bae2eb6caea7487c9f110edff1b75 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 17:19:26 -0400 Subject: [PATCH 16/73] remove price updater from daemon --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index fb510a0cc..c6a895f94 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -174,7 +174,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.streams = {} self.known_dht_nodes = KNOWN_DHT_NODES self.first_run_after_update = False - self.last_traded_rate = None self.uploaded_temp_files = [] if os.name == "nt": @@ -363,7 +362,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker = LoopingCall(self._check_network_connection) self.version_checker = LoopingCall(self._check_remote_versions) self.connection_problem_checker = LoopingCall(self._check_connection_problems) - self.price_checker = LoopingCall(self._update_exchange) # self.lbrynet_connection_checker = LoopingCall(self._check_lbrynet_connection) self.sd_identifier = StreamDescriptorIdentifier() @@ -497,7 +495,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) self.connection_problem_checker.start(1) - self.price_checker.start(600) if host_ui: self.lbry_ui_manager.update_checker.start(1800, now=False) @@ -612,13 +609,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): if not self.connected_to_internet: self.connection_problem = CONNECTION_PROBLEM_CODES[1] - def _update_exchange(self): - try: - r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'}) - self.last_traded_rate = float(json.loads(r.text)['result']['Last']) - except: - self.last_traded_rate = None - def _start_server(self): if self.peer_port is not None: @@ -737,8 +727,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.connection_problem_checker.stop() if self.lbry_ui_manager.update_checker.running: self.lbry_ui_manager.update_checker.stop() - if self.price_checker.running: - self.price_checker.stop() self._clean_up_temp_files() From 0811ebb52d6b7bdbda6a949c914410f1b2b5e04e Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 17:20:51 -0400 Subject: [PATCH 17/73] fix key payment problems --- lbrynet/core/LBRYMetadata.py | 81 +++++++++++++----------- lbrynet/lbrynet_daemon/LBRYDaemon.py | 19 +++--- lbrynet/lbrynet_daemon/LBRYDownloader.py | 25 ++++---- 3 files changed, 67 insertions(+), 58 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 133c34482..e57ee1ad6 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -2,11 +2,13 @@ import requests import json import time +from copy import deepcopy from googlefinance import getQuotes from lbrynet.conf import CURRENCIES import logging log = logging.getLogger(__name__) +log.setLevel(logging.INFO) BITTREX_FEE = 0.0025 @@ -26,23 +28,42 @@ FEE_REVISIONS = {'0.0.1': {'required': ['amount', 'address'], 'optional': []}} CURRENT_FEE_REVISION = '0.0.1' -class LBRYFee(object): - def __init__(self, fee_dict, rate_dict): - fee = LBRYFeeFormat(fee_dict) - - for currency in fee: - self.address = fee[currency]['address'] - if not isinstance(fee[currency]['amount'], float): - self.amount = float(fee[currency]['amount']) - else: - self.amount = fee[currency]['amount'] +class LBRYFeeFormat(dict): + def __init__(self, fee_dict): + dict.__init__(self) + self.fee_version = None + f = deepcopy(fee_dict) + assert len(fee_dict) == 1 + for currency in fee_dict: + assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) self.currency_symbol = currency + self.update({currency: {}}) + for version in FEE_REVISIONS: + for k in FEE_REVISIONS[version]['required']: + assert k in fee_dict[currency], "Missing required fee field: %s" % k + self[currency].update({k: f[currency].pop(k)}) + for k in FEE_REVISIONS[version]['optional']: + if k in fee_dict[currency]: + self[currency].update({k: f[currency].pop(k)}) + if not len(f): + self.fee_version = version + break + assert f[currency] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) - assert 'BTCLBC' in rate_dict and 'USDBTC' in rate_dict + self.amount = self[self.currency_symbol]['amount'] if isinstance(self[self.currency_symbol]['amount'], float) else float(self[self.currency_symbol]['amount']) + self.address = self[self.currency_symbol]['address'] + + +class LBRYFee(LBRYFeeFormat): + def __init__(self, fee_dict, rate_dict): + LBRYFeeFormat.__init__(self, fee_dict) + rates = deepcopy(rate_dict) + + assert 'BTCLBC' in rates and 'USDBTC' in rates for fx in rate_dict: - assert int(time.time()) - int(rate_dict[fx]['ts']) < 3600, "%s quote is out of date" % fx - self._USDBTC = {'spot': rate_dict['USDBTC']['spot'], 'ts': rate_dict['USDBTC']['ts']} - self._BTCLBC = {'spot': rate_dict['BTCLBC']['spot'], 'ts': rate_dict['BTCLBC']['ts']} + assert int(time.time()) - int(rates[fx]['ts']) < 3600, "%s quote is out of date" % fx + self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']} + self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']} def to_lbc(self): r = None @@ -67,45 +88,31 @@ class LBRYFee(object): return r def _usd_to_btc(self, usd): + # log.error("usd to btc: " + str(usd)) + # log.error("%f * %f = %f" % (self._USDBTC['spot'], float(usd), self._USDBTC['spot'] * float(usd))) return self._USDBTC['spot'] * float(usd) def _btc_to_usd(self, btc): + # log.error("btc to usd: " + str(btc)) + # log.error("%f / %f = %f" % (float(btc), self._USDBTC['spot'], float(btc) / self._USDBTC['spot'])) return float(btc) / self._USDBTC['spot'] def _btc_to_lbc(self, btc): + # log.error("btc to lbc: " + str(btc)) + # log.error("%f * %f = %f" % (float(btc), self._BTCLBC['spot'], float(btc) * self._BTCLBC['spot'] / (1.0 - BITTREX_FEE))) return float(btc) * self._BTCLBC['spot'] / (1.0 - BITTREX_FEE) def _lbc_to_btc(self, lbc): + # log.error("lbc to btc: " + str(lbc)) + # log.error("%f / %f = %f" % (self._BTCLBC['spot'], float(lbc), self._BTCLBC['spot'] / float(lbc))) return self._BTCLBC['spot'] / float(lbc) -class LBRYFeeFormat(dict): - def __init__(self, fee_dict): - dict.__init__(self) - self.fee_version = None - f = fee_dict.copy() - assert len(fee_dict) == 1 - for currency in fee_dict: - assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) - self.update({currency: {}}) - for version in FEE_REVISIONS: - for k in FEE_REVISIONS[version]['required']: - assert k in fee_dict[currency], "Missing required fee field: %s" % k - self[currency].update({k: f[currency].pop(k)}) - for k in FEE_REVISIONS[version]['optional']: - if k in fee_dict[currency]: - self[currency].update({k: f[currency].pop(k)}) - if not len(f): - self.fee_version = version - break - assert f[currency] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) - - class Metadata(dict): def __init__(self, metadata): dict.__init__(self) self.metaversion = None - m = metadata.copy() + m = deepcopy(metadata) assert "sources" in metadata, "No sources given" for source in metadata['sources']: diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index c6a895f94..518dbdce6 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1899,20 +1899,19 @@ class LBRYDaemon(jsonrpc.JSONRPC): file_path = p['file_path'] metadata = p['metadata'] - def _set_address(address): - metadata['fee']['address'] = address + d = defer.succeed(None) + + def _set_address(address, currency): + metadata['fee'][currency]['address'] = address return defer.succeed(None) if 'fee' in p: metadata['fee'] = p['fee'] - if 'address' not in metadata['fee']: - d = self.session.wallet.get_new_address() - d.addCallback(_set_address) - else: - d = defer.succeed(None) - else: - d = defer.succeed(None) - + assert len(metadata['fee']) == 1, "Too many fees" + for c in metadata['fee']: + if 'address' not in metadata['fee'][c]: + d = self.session.wallet.get_new_address() + d.addCallback(lambda addr: _set_address(addr, c)) pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 8d25aa56f..00c137869 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -3,6 +3,7 @@ import logging import os import sys +from copy import deepcopy from appdirs import user_data_dir from datetime import datetime from twisted.internet import defer @@ -58,7 +59,7 @@ class GetStream(object): self.sd_identifier = sd_identifier self.stream_hash = None self.max_key_fee = max_key_fee - self.metadata = None + self.stream_info = None self.stream_info_manager = None self.d = defer.Deferred(None) self.timeout = timeout @@ -86,20 +87,23 @@ class GetStream(object): def start(self, stream_info, name): self.resolved_name = name - self.metadata = stream_info - self.stream_hash = self.metadata['sources']['lbry_sd_hash'] + self.stream_info = deepcopy(stream_info) + self.description = self.stream_info['description'] + self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] - if 'fee' in self.metadata: - self.fee = LBRYFee(self.metadata['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + if 'fee' in self.stream_info: + self.fee = LBRYFee(self.stream_info['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) if isinstance(self.max_key_fee, float): if self.fee.to_lbc() > self.max_key_fee: log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) return defer.fail(KeyFeeAboveMaxAllowed()) + log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) elif isinstance(self.max_key_fee, dict): - max_key = LBRYFee(self.max_key_fee, {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + max_key = LBRYFee(deepcopy(self.max_key_fee), {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) if self.fee.to_lbc() > max_key.to_lbc(): log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), max_key.to_lbc(), self.resolved_name)) return defer.fail(KeyFeeAboveMaxAllowed()) + log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), max_key.to_lbc(), self.resolved_name)) def _cause_timeout(): self.timeout_counter = self.timeout * 2 @@ -130,20 +134,19 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): if self.fee is not None: - fee_lbc = self.fee.to_lbc() + fee_lbc = float(self.fee.to_lbc()) reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) if reserved_points is None: return defer.fail(InsufficientFundsError()) - log.info("Key fee: %f --> %s" % (fee_lbc, self.fee.address)) - d = self.wallet.send_points_to_address(reserved_points, self.fee.address) - return d + return self.wallet.send_points_to_address(reserved_points, fee_lbc) return defer.succeed(None) + d = _pay_key_fee() + self.downloader = downloader self.download_path = os.path.join(downloader.download_directory, downloader.file_name) - d = _pay_key_fee() d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name)) d.addCallback(lambda _: self.downloader.start()) From 3d88eb92c190d26693691f779f911fcbf936433e Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 17:21:03 -0400 Subject: [PATCH 18/73] change default max fee to $25 --- lbrynet/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index cb626d717..16a459c03 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -42,7 +42,7 @@ DEFAULT_WALLET = "lbryum" WALLET_TYPES = ["lbryum", "lbrycrd"] DEFAULT_TIMEOUT = 30 DEFAULT_MAX_SEARCH_RESULTS = 25 -DEFAULT_MAX_KEY_FEE = {'BTC': {'amount': 0.025}} +DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}} DEFAULT_SEARCH_TIMEOUT = 3.0 DEFAULT_CACHE_TIME = 3600 DEFAULT_UI_BRANCH = "master" From f82cffe882a4694de6c937928bfa791ab4e33f16 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 17:21:41 -0400 Subject: [PATCH 19/73] verbose logging --- lbrynet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 969401685..8d1db90a5 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -2,7 +2,7 @@ import logging log = logging.getLogger(__name__) logging.getLogger(__name__).addHandler(logging.NullHandler()) -log.setLevel(logging.ERROR) +log.setLevel(logging.INFO) __version__ = "0.3.10" version = tuple(__version__.split('.')) \ No newline at end of file From d86e709b90263c3d95b2a9b9b96234e1397993a2 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 17:24:59 -0400 Subject: [PATCH 20/73] reduce min blob prices --- lbrynet/conf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 16a459c03..71df9f8fd 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -10,10 +10,11 @@ MAX_RESPONSE_INFO_SIZE = 2**16 MAX_BLOB_INFOS_TO_REQUEST = 20 BLOBFILES_DIR = ".blobfiles" BLOB_SIZE = 2**21 -MIN_BLOB_DATA_PAYMENT_RATE = .005 # points/megabyte -MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos -MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos -MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05 # points/1000 infos + +MIN_BLOB_DATA_PAYMENT_RATE = .0005 # points/megabyte +MIN_BLOB_INFO_PAYMENT_RATE = .002 # points/1000 infos +MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .005 # points/1000 infos +MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .005 # points/1000 infos MAX_CONNECTIONS_PER_STREAM = 5 KNOWN_DHT_NODES = [('104.236.42.182', 4000), From 5e1fdf66b0fa8be029203bb3374c18414f17e57a Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 18:18:34 -0400 Subject: [PATCH 21/73] fix download problem --- lbrynet/conf.py | 8 +++--- lbrynet/lbrynet_daemon/LBRYDownloader.py | 31 ++++++++++++++++-------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 71df9f8fd..85a9f6836 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -11,10 +11,10 @@ MAX_BLOB_INFOS_TO_REQUEST = 20 BLOBFILES_DIR = ".blobfiles" BLOB_SIZE = 2**21 -MIN_BLOB_DATA_PAYMENT_RATE = .0005 # points/megabyte -MIN_BLOB_INFO_PAYMENT_RATE = .002 # points/1000 infos -MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .005 # points/1000 infos -MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .005 # points/1000 infos +MIN_BLOB_DATA_PAYMENT_RATE = .005 # points/megabyte +MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos +MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos +MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05 # points/1000 infos MAX_CONNECTIONS_PER_STREAM = 5 KNOWN_DHT_NODES = [('104.236.42.182', 4000), diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 00c137869..909a3f340 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -67,7 +67,7 @@ class GetStream(object): self.download_directory = download_directory self.download_path = None self.downloader = None - self.finished = defer.Deferred() + self.finished = defer.Deferred(None) self.checker = LoopingCall(self.check_status) self.code = STREAM_STAGES[0] @@ -105,7 +105,9 @@ class GetStream(object): return defer.fail(KeyFeeAboveMaxAllowed()) log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), max_key.to_lbc(), self.resolved_name)) - def _cause_timeout(): + def _cause_timeout(err): + log.error(err) + log.debug('Forcing a timeout') self.timeout_counter = self.timeout * 2 def _set_status(x, status): @@ -113,20 +115,29 @@ class GetStream(object): self.code = next(s for s in STREAM_STAGES if s[0] == status) return x + def get_downloader_factory(metadata): + for factory in metadata.factories: + if isinstance(factory, ManagedLBRYFileDownloaderFactory): + return factory, metadata + raise Exception('No suitable factory was found in {}'.format(metadata.factories)) + + def make_downloader(args): + factory, metadata = args + return factory.make_downloader(metadata, + [self.data_rate, True], + self.payment_rate_manager, + download_directory=self.download_directory, + file_name=self.file_name) + self.checker.start(1) self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE)) self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager)) self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE)) - self.d.addCallback(lambda metadata: (next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)), - metadata)) - self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata, - [self.data_rate, True], - self.payment_rate_manager, - download_directory=self.download_directory, - file_name=self.file_name)) - self.d.addCallbacks(self._start_download, lambda _: _cause_timeout()) + self.d.addCallback(get_downloader_factory) + self.d.addCallback(make_downloader) + self.d.addCallbacks(self._start_download, _cause_timeout) self.d.callback(None) return self.finished From bd6fa35d4a3d734ae5b5f071b5fc57782f568716 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 18:19:40 -0400 Subject: [PATCH 22/73] disable verbose dht error --- lbrynet/dht/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index a3bf56553..ff77bdfbf 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -240,9 +240,9 @@ class Node(object): known_nodes = {} def log_error(err, n): - log.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n)) - log.error(binascii.hexlify(err.getErrorMessage())) - log.error(err.getTraceback()) + log.debug("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n)) + log.debug(binascii.hexlify(err.getErrorMessage())) + log.debug(err.getTraceback()) def log_success(res): log.debug("Response to store request: %s", str(res)) From ee9e8b365c2c6ab5f2a5e19768880ac30b374c44 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 18:42:51 -0400 Subject: [PATCH 23/73] add debug log line --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 518dbdce6..aedd5c5c6 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1902,6 +1902,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = defer.succeed(None) def _set_address(address, currency): + log.info("Generated new address for key fee: " + str(address)) metadata['fee'][currency]['address'] = address return defer.succeed(None) From fe39901885fb34df525010e16b6be2997bbb1b75 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 20:52:05 -0400 Subject: [PATCH 24/73] update claim when publishing to an already claimed name -also save name_metadata for each unique claim rather than for each name --- lbrynet/core/LBRYWallet.py | 23 ++++++++---------- lbrynet/lbrynet_daemon/LBRYDaemon.py | 31 +++++-------------------- lbrynet/lbrynet_daemon/LBRYPublisher.py | 16 +++++++++---- 3 files changed, 28 insertions(+), 42 deletions(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index aa9df4127..b2848693a 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -380,7 +380,7 @@ class LBRYWallet(object): return Failure(InvalidStreamInfoError(name)) m = Metadata(value_dict) if 'txid' in result: - d = self._save_name_metadata(name, m['sources']['lbry_sd_hash'], str(result['txid'])) + d = self._save_name_metadata(name, str(result['txid']), m['sources']['lbry_sd_hash']) d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, m.metaversion))) d.addCallback(lambda _: m) return d @@ -398,7 +398,8 @@ class LBRYWallet(object): d = self._send_name_claim(name, json.dumps(metadata), bid) def _save_metadata(txid): - d = self._save_name_metadata(name, metadata['sources']['lbry_sd_hash'], txid) + log.info("Saving metadata for claim %s" % txid) + d = self._save_name_metadata(name, txid, metadata['sources']['lbry_sd_hash']) d.addCallback(lambda _: txid) return d @@ -443,10 +444,12 @@ class LBRYWallet(object): d.addCallback(self._get_decoded_tx) return d - def update_name(self, name, value, amount): + def update_name(self, name, bid, value, old_txid): d = self._get_value_for_name(name) - d.addCallback(lambda r: (self._update_name(r['txid'], json.dumps(value), amount), r['txid'])) - d.addCallback(lambda (new_txid, old_txid): self._update_name_metadata(name, value['sources']['lbry_sd_hash'], old_txid, new_txid)) + d.addCallback(lambda r: self.abandon_name(r['txid'] if not old_txid else old_txid)) + d.addCallback(lambda r: log.info("Abandon claim tx %s" % str(r))) + d.addCallback(lambda _: self.claim_name(name, bid, value)) + return d def get_name_and_validity_for_sd_hash(self, sd_hash): @@ -556,19 +559,13 @@ class LBRYWallet(object): " txid text, " + " sd_hash text)") - def _save_name_metadata(self, name, sd_hash, txid): - d = self.db.runQuery("select * from name_metadata where txid=?", (txid,)) + def _save_name_metadata(self, name, txid, sd_hash): + d = self.db.runQuery("select * from name_metadata where name=? and txid=? and sd_hash=?", (name, txid, sd_hash)) d.addCallback(lambda r: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, txid, sd_hash)) if not len(r) else None) return d - def _update_name_metadata(self, name, sd_hash, old_txid, new_txid): - d = self.db.runQuery("delete * from name_metadata where txid=? and sd_hash=?", (old_txid, sd_hash)) - d.addCallback(lambda _: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, new_txid, sd_hash))) - d.addCallback(lambda _: new_txid) - return d - def _get_claim_metadata_for_sd_hash(self, sd_hash): d = self.db.runQuery("select name, txid from name_metadata where sd_hash=?", (sd_hash,)) d.addCallback(lambda r: r[0] if len(r) else None) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index aedd5c5c6..e7b95bdf4 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1899,24 +1899,26 @@ class LBRYDaemon(jsonrpc.JSONRPC): file_path = p['file_path'] metadata = p['metadata'] - d = defer.succeed(None) + update = False def _set_address(address, currency): log.info("Generated new address for key fee: " + str(address)) metadata['fee'][currency]['address'] = address return defer.succeed(None) + d = defer.succeed(None) + if 'fee' in p: metadata['fee'] = p['fee'] assert len(metadata['fee']) == 1, "Too many fees" for c in metadata['fee']: if 'address' not in metadata['fee'][c]: - d = self.session.wallet.get_new_address() + d.addCallback(lambda _: self.session.wallet.get_new_address()) d.addCallback(lambda addr: _set_address(addr, c)) pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) - - d.addCallback(lambda _: pub.start(name, file_path, bid, metadata)) + d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) + d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r.txid)) d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) @@ -2181,27 +2183,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - def jsonrpc_update_name(self, p): - """ - Update name claim - - Args: - 'name': the uri of the claim to be updated - 'metadata': new metadata dict - 'amount': bid amount of updated claim - Returns: - txid - """ - - name = p['name'] - metadata = p['metadata'] if isinstance(p['metadata'], dict) else json.loads(p['metadata']) - amount = p['amount'] - - d = self.session.wallet.update_name(name, metadata, amount) - d.addCallback(lambda r: self._render_response(r, OK_CODE)) - - return d - def jsonrpc_log(self, p): """ Log message diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 10f787209..7ff284577 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -45,7 +45,7 @@ class Publisher(object): self.txid = None self.metadata = {} - def start(self, name, file_path, bid, metadata): + def start(self, name, file_path, bid, metadata, old_txid=None): def _show_result(): log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) @@ -55,6 +55,7 @@ class Publisher(object): self.file_path = file_path self.bid_amount = bid self.metadata = metadata + self.old_txid = old_txid d = self._check_file_path(self.file_path) d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager, @@ -102,9 +103,16 @@ class Publisher(object): self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name))[0] self.metadata['ver'] = CURRENT_METADATA_VERSION - d = self.wallet.claim_name(self.publish_name, - self.bid_amount, - Metadata(self.metadata)) + + if self.old_txid: + d = self.wallet.update_name(self.publish_name, + self.bid_amount, + Metadata(self.metadata), + self.old_txid) + else: + d = self.wallet.claim_name(self.publish_name, + self.bid_amount, + Metadata(self.metadata)) def set_tx_hash(txid): self.txid = txid From 3e602bce529ee143fe56c8b30d94037961013794 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 20:58:37 -0400 Subject: [PATCH 25/73] fix publish callback --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index e7b95bdf4..b66147b64 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1918,7 +1918,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) - d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r.txid)) + d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, None if r is None else r.txid)) d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) From 29b5aef73a8b491c2218bcf0b58e4632bca97d84 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 21:24:58 -0400 Subject: [PATCH 26/73] call restore() after claiming name --- lbrynet/lbrynet_daemon/LBRYPublisher.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 7ff284577..4d27a6973 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -43,6 +43,7 @@ class Publisher(object): self.verified = False self.lbry_file = None self.txid = None + self.stream_hash = None self.metadata = {} def start(self, name, file_path, bid, metadata, old_txid=None): @@ -63,6 +64,7 @@ class Publisher(object): d.addCallback(self.add_to_lbry_files) d.addCallback(lambda _: self._create_sd_blob()) d.addCallback(lambda _: self._claim_name()) + d.addCallback(lambda _: self.set_status()) d.addCallbacks(lambda _: _show_result(), self._show_publish_error) return d @@ -75,16 +77,15 @@ class Publisher(object): return True return threads.deferToThread(check_file_threaded) - def set_status(self, lbry_file_downloader): + def set_lbry_file(self, lbry_file_downloader): self.lbry_file = lbry_file_downloader - d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED) - d.addCallback(lambda _: lbry_file_downloader.restore()) - return d + return defer.succeed(None) def add_to_lbry_files(self, stream_hash): + self.stream_hash = stream_hash prm = PaymentRateManager(self.session.base_payment_rate_manager) d = self.lbry_file_manager.add_lbry_file(stream_hash, prm) - d.addCallback(self.set_status) + d.addCallback(self.set_lbry_file) return d def _create_sd_blob(self): @@ -99,6 +100,11 @@ class Publisher(object): d.addCallback(set_sd_hash) return d + def set_status(self): + d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED) + d.addCallback(lambda _: self.lbry_file.restore()) + return d + def _claim_name(self): self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name))[0] From 22f73a081769984f2e637f1e2dfb8a5850aa4e81 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 26 Jul 2016 21:46:04 -0400 Subject: [PATCH 27/73] add force parameter to resolve_name --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index b66147b64..8c04cf345 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1671,12 +1671,17 @@ class LBRYDaemon(jsonrpc.JSONRPC): metadata from name claim """ - if 'name' in p.keys(): + if 'force' in p: + force = p['force'] + else: + force = False + + if 'name' in p: name = p['name'] else: return self._render_response(None, BAD_REQUEST) - d = self._resolve_name(name) + d = self._resolve_name(name, force_refresh=force) d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d From 5abeceac1fd39e147caa0955173ed58bc58e1c6c Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 01:13:41 -0400 Subject: [PATCH 28/73] abandon previous claim in update --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 11 +++++++++-- lbrynet/lbrynet_daemon/LBRYPublisher.py | 12 +++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 8c04cf345..6ecafb741 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1911,7 +1911,13 @@ class LBRYDaemon(jsonrpc.JSONRPC): metadata['fee'][currency]['address'] = address return defer.succeed(None) - d = defer.succeed(None) + def _delete_data(lbry_file): + txid = lbry_file.txid + d = self._delete_lbry_file(lbry_file, delete_file=False) + d.addCallback(lambda _: txid) + return d + + d = self._resolve_name(name, force_refresh=True) if 'fee' in p: metadata['fee'] = p['fee'] @@ -1923,7 +1929,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) - d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, None if r is None else r.txid)) + d.addCallback(lambda l: None if not l else _delete_data(l)) + d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r)) d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 4d27a6973..f27e6e48b 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -46,7 +46,7 @@ class Publisher(object): self.stream_hash = None self.metadata = {} - def start(self, name, file_path, bid, metadata, old_txid=None): + def start(self, name, file_path, bid, metadata, old_txid): def _show_result(): log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) @@ -111,10 +111,12 @@ class Publisher(object): self.metadata['ver'] = CURRENT_METADATA_VERSION if self.old_txid: - d = self.wallet.update_name(self.publish_name, - self.bid_amount, - Metadata(self.metadata), - self.old_txid) + + d = self.wallet.abandon_name(self.old_txid) + d.addCallback(lambda tx: log.info("Abandoned tx %s" % str(tx))) + d.addCallback(lambda _: self.wallet.claim_name(self.publish_name, + self.bid_amount, + Metadata(self.metadata))) else: d = self.wallet.claim_name(self.publish_name, self.bid_amount, From f3af7a6b77c2dfebff36082fff910aa153ecf0d8 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 01:39:23 -0400 Subject: [PATCH 29/73] add errback for unknown name --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 6ecafb741..3e814c401 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1918,6 +1918,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d d = self._resolve_name(name, force_refresh=True) + d.addErrback(lambda _: None) if 'fee' in p: metadata['fee'] = p['fee'] From 2102776be81d1eb5a5a971293f668f35da18c0f6 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 01:44:01 -0400 Subject: [PATCH 30/73] another errback --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 3e814c401..237bdadad 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1904,8 +1904,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): file_path = p['file_path'] metadata = p['metadata'] - update = False - def _set_address(address, currency): log.info("Generated new address for key fee: " + str(address)) metadata['fee'][currency]['address'] = address @@ -1930,7 +1928,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) - d.addCallback(lambda l: None if not l else _delete_data(l)) + d.addCallbacks(lambda l: None if not l else _delete_data(l), lambda _: None) d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r)) d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) From ef580a7680b5316b240662c38dff58602d9b7a24 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 01:51:09 -0400 Subject: [PATCH 31/73] errback for pending of date lbry_file --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 237bdadad..c1206a030 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1311,7 +1311,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): if f.txid: d = self._resolve_name(f.uri) - d.addCallback(_add_to_dict) + d.addCallbacks(_add_to_dict, lambda _: _add_to_dict("Pending confirmation")) else: d = defer.succeed(message) return d From 4604e9cd5488919b359063d5ff9a32bda1e6a82d Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 03:00:44 -0400 Subject: [PATCH 32/73] check pending claims after one is made --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 30 ++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index c1206a030..a311555d3 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -172,6 +172,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.session = None self.waiting_on = {} self.streams = {} + self.pending_claims = {} self.known_dht_nodes = KNOWN_DHT_NODES self.first_run_after_update = False self.uploaded_temp_files = [] @@ -362,6 +363,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker = LoopingCall(self._check_network_connection) self.version_checker = LoopingCall(self._check_remote_versions) self.connection_problem_checker = LoopingCall(self._check_connection_problems) + self.pending_claim_checker = LoopingCall(self._check_pending_claims) # self.lbrynet_connection_checker = LoopingCall(self._check_lbrynet_connection) self.sd_identifier = StreamDescriptorIdentifier() @@ -609,6 +611,24 @@ class LBRYDaemon(jsonrpc.JSONRPC): if not self.connected_to_internet: self.connection_problem = CONNECTION_PROBLEM_CODES[1] + def _add_to_pending_claims(self, name, txid): + log.info("Adding lbry://%s to pending claims, txid %s" % (name, txid)) + self.pending_claims[name] = txid + return txid + + def _check_pending_claims(self): + def _start_file(name): + d = defer.succeed(self.pending_claims.pop(name)) + d.addCallback(lambda _: self._get_lbry_file("name", name, return_json=False)) + d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running") + + for name in self.pending_claims: + log.info("Checking if new claim for lbry://%s is confirmed" % name) + d = self._resolve_name(name, force_refresh=True) + d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) + d.addCallbacks(lambda lbry_file: _start_file(name) if self.pending_claims[name] == lbry_file['txid'] and not isinstance(lbry_file['metadata'], str) else self._add_to_pending_claims(name, self.pending_claims.pop(name)), + self._add_to_pending_claims(name, self.pending_claims.pop(name))) + def _start_server(self): if self.peer_port is not None: @@ -727,6 +747,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.connection_problem_checker.stop() if self.lbry_ui_manager.update_checker.running: self.lbry_ui_manager.update_checker.stop() + if self.pending_claim_checker.running: + self.pending_claim_checker.stop() self._clean_up_temp_files() @@ -1915,6 +1937,9 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: txid) return d + if not self.pending_claim_checker.running: + self.pending_claim_checker.start(30) + d = self._resolve_name(name, force_refresh=True) d.addErrback(lambda _: None) @@ -1930,8 +1955,9 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) d.addCallbacks(lambda l: None if not l else _delete_data(l), lambda _: None) d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r)) - d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), - lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) + d.addCallback(lambda txid: self._add_to_pending_claims(name, txid)) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + d.addErrback(lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) return d From 0f602e0e62db636f0ce875635c964845dfa19a7e Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 27 Jul 2016 11:59:57 -0500 Subject: [PATCH 33/73] allow bittrex fee to be changed --- lbrynet/core/LBRYMetadata.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index e57ee1ad6..106fabe1e 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -55,8 +55,9 @@ class LBRYFeeFormat(dict): class LBRYFee(LBRYFeeFormat): - def __init__(self, fee_dict, rate_dict): + def __init__(self, fee_dict, rate_dict, bittrex_fee=None): LBRYFeeFormat.__init__(self, fee_dict) + self.bittrex_fee = BITTREX_FEE if bittrex_fee is None else bittrex_fee rates = deepcopy(rate_dict) assert 'BTCLBC' in rates and 'USDBTC' in rates @@ -88,23 +89,15 @@ class LBRYFee(LBRYFeeFormat): return r def _usd_to_btc(self, usd): - # log.error("usd to btc: " + str(usd)) - # log.error("%f * %f = %f" % (self._USDBTC['spot'], float(usd), self._USDBTC['spot'] * float(usd))) return self._USDBTC['spot'] * float(usd) def _btc_to_usd(self, btc): - # log.error("btc to usd: " + str(btc)) - # log.error("%f / %f = %f" % (float(btc), self._USDBTC['spot'], float(btc) / self._USDBTC['spot'])) return float(btc) / self._USDBTC['spot'] def _btc_to_lbc(self, btc): - # log.error("btc to lbc: " + str(btc)) - # log.error("%f * %f = %f" % (float(btc), self._BTCLBC['spot'], float(btc) * self._BTCLBC['spot'] / (1.0 - BITTREX_FEE))) - return float(btc) * self._BTCLBC['spot'] / (1.0 - BITTREX_FEE) + return float(btc) * self._BTCLBC['spot'] / (1.0 - self.bittrex_fee) def _lbc_to_btc(self, lbc): - # log.error("lbc to btc: " + str(lbc)) - # log.error("%f / %f = %f" % (self._BTCLBC['spot'], float(lbc), self._BTCLBC['spot'] / float(lbc))) return self._BTCLBC['spot'] / float(lbc) From 6a1efedc1fb4ddf89079e6b2c7185448feeb1100 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 27 Jul 2016 12:05:29 -0500 Subject: [PATCH 34/73] some example tests. please expand on these --- tests/lbrynet/core/test_LBRYMetadata.py | 102 ++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/lbrynet/core/test_LBRYMetadata.py diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py new file mode 100644 index 000000000..ac88ffa77 --- /dev/null +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -0,0 +1,102 @@ +import mock +from lbrynet.core import LBRYMetadata + +from twisted.trial import unittest + + +class LBRYFeeFormatTest(unittest.TestCase): + def test_fee_created_with_correct_inputs(self): + fee_dict = { + 'USD': { + 'amount': 10, + 'address': None + } + } + fee = LBRYMetadata.LBRYFeeFormat(fee_dict) + self.assertEqual(10, fee['USD']['amount']) + + +class LBRYFeeTest(unittest.TestCase): + def setUp(self): + self.patcher = mock.patch('time.time') + self.time = self.patcher.start() + self.time.return_value = 0 + + def tearDown(self): + self.time.stop() + + def test_fee_converts_to_lbc(self): + fee_dict = { + 'USD': { + 'amount': 10, + 'address': None + } + } + rates = {'BTCLBC': {'spot': 3, 'ts': 2}, 'USDBTC': {'spot': 2, 'ts': 3}} + fee = LBRYMetadata.LBRYFee(fee_dict, rates, 0) + self.assertEqual(60, fee.to_lbc()) + + +class MetadataTest(unittest.TestCase): + def test_assertion_if_source_is_missing(self): + metadata = {} + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_assertion_if_invalid_source(self): + metadata = { + 'sources': {'garbage': None} + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_assertion_if_missing_v001_field(self): + metadata = { + 'sources': [], + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_version_is_001_if_all_fields_are_present(self): + metadata = { + 'sources': [], + 'title': None, + 'description': None, + 'author': None, + 'language': None, + 'license': None, + 'content-type': None, + } + m = LBRYMetadata.Metadata(metadata) + self.assertEquals('0.0.1', m.metaversion) + + def test_assertion_if_there_is_an_extra_field(self): + metadata = { + 'sources': [], + 'title': None, + 'description': None, + 'author': None, + 'language': None, + 'license': None, + 'content-type': None, + 'extra': None + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_version_is_002_if_all_fields_are_present(self): + metadata = { + 'sources': [], + 'title': None, + 'description': None, + 'author': None, + 'language': None, + 'license': None, + 'content-type': None, + 'nsfw': None, + 'ver': None + } + m = LBRYMetadata.Metadata(metadata) + self.assertEquals('0.0.2', m.metaversion) + + From 900d3ad01d12ca5e39783eaac79802324efb8961 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 27 Jul 2016 13:12:20 -0500 Subject: [PATCH 35/73] Fixes errback not being callable Small code refactoring to make it more obvious what both the callback and errback are doing --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 2408f855b..e5ced6588 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -609,12 +609,27 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._get_lbry_file("name", name, return_json=False)) d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running") + def _process_lbry_file(name, lbry_file): + ready_to_start = ( + self.pending_claims[name] == lbry_file['txid'] and + not isinstance(lbry_file['metadata'], str) + ) + if ready_to_start: + _start_file(name) + else: + re_add_to_pending_claims(name) + + def re_add_to_pending_claims(name): + self._add_to_pending_claims(name, self.pending_claims.pop(name)), + for name in self.pending_claims: log.info("Checking if new claim for lbry://%s is confirmed" % name) d = self._resolve_name(name, force_refresh=True) d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) - d.addCallbacks(lambda lbry_file: _start_file(name) if self.pending_claims[name] == lbry_file['txid'] and not isinstance(lbry_file['metadata'], str) else self._add_to_pending_claims(name, self.pending_claims.pop(name)), - self._add_to_pending_claims(name, self.pending_claims.pop(name))) + d.addCallbacks( + lambda lbry_file: _process_lbry_file(name, lbry_file), + lambda _: re_add_to_pending_claims(name) + ) def _start_server(self): if self.peer_port is not None: From 0925171e0795421a2d7be997be2844a07ec7d091 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 27 Jul 2016 13:59:26 -0500 Subject: [PATCH 36/73] bug fix: use txid attribute, not key --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index e5ced6588..20f0577f3 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -610,8 +610,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running") def _process_lbry_file(name, lbry_file): + # lbry_file is an instance of ManagedLBRYFileDownloader or None ready_to_start = ( - self.pending_claims[name] == lbry_file['txid'] and + lbry_file and + self.pending_claims[name] == lbry_file.txid and not isinstance(lbry_file['metadata'], str) ) if ready_to_start: @@ -620,7 +622,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): re_add_to_pending_claims(name) def re_add_to_pending_claims(name): - self._add_to_pending_claims(name, self.pending_claims.pop(name)), + txid = self.pending_claims.pop(name) + self._add_to_pending_claims(name, txid) for name in self.pending_claims: log.info("Checking if new claim for lbry://%s is confirmed" % name) From 7f5560de6dcc5f8ca21eff8f53cdc979c72a2fdd Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 15:02:19 -0400 Subject: [PATCH 37/73] comment spacing and removing setLevel --- lbrynet/core/LBRYMetadata.py | 8 +++----- lbrynet/core/LBRYWallet.py | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index e57ee1ad6..5970fba3f 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -8,7 +8,6 @@ from lbrynet.conf import CURRENCIES import logging log = logging.getLogger(__name__) -log.setLevel(logging.INFO) BITTREX_FEE = 0.0025 @@ -16,14 +15,13 @@ SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] -#v0.0.1 metadata +# v0.0.1 metadata METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}} -#v0.0.2 metadata additions +# v0.0.2 metadata additions METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']} CURRENT_METADATA_VERSION = '0.0.2' - -#v0.0.1 fee +# v0.0.1 fee FEE_REVISIONS = {'0.0.1': {'required': ['amount', 'address'], 'optional': []}} CURRENT_FEE_REVISION = '0.0.1' diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 21f08bba1..3e7573a6e 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -32,7 +32,6 @@ from lbrynet.conf import SOURCE_TYPES from lbrynet.core.LBRYMetadata import Metadata log = logging.getLogger(__name__) -log.setLevel(logging.INFO) alert = logging.getLogger("lbryalert." + __name__) From e4ebbd8f6982b699d2bd8eb14d6a0abd28ec0a51 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 15:02:54 -0400 Subject: [PATCH 38/73] neater force param in resolve_name --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 2408f855b..45e7f678b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1680,10 +1680,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): metadata from name claim """ - if 'force' in p: - force = p['force'] - else: - force = False + force = p.get('force', False) if 'name' in p: name = p['name'] From f6d91280d7a8394e0040a024b3340be4e79dbcdd Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 27 Jul 2016 15:03:03 -0400 Subject: [PATCH 39/73] remove debug print line --- lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index ba807835e..3f2107079 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -67,7 +67,6 @@ class LBRYDaemonRequest(server.Request): from twisted.web.http import parse_qs if self.do_log: print '%f Request Received' % time.time() - print self.content self.content.seek(0,0) self.args = {} From 4ade80c8a93124efac90e0110bc67b29cda9aae9 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 02:19:45 -0400 Subject: [PATCH 40/73] clean up metadata and fee --- lbrynet/core/LBRYMetadata.py | 97 +++++++++++++++--------- lbrynet/core/LBRYWallet.py | 2 +- lbrynet/lbrynet_daemon/LBRYDownloader.py | 43 ++++++----- tests/lbrynet/core/test_LBRYMetadata.py | 6 +- 4 files changed, 89 insertions(+), 59 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index b3d80cb91..beed9bf12 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -26,35 +26,50 @@ FEE_REVISIONS = {'0.0.1': {'required': ['amount', 'address'], 'optional': []}} CURRENT_FEE_REVISION = '0.0.1' -class LBRYFeeFormat(dict): +class LBRYFeeValidator(dict): def __init__(self, fee_dict): dict.__init__(self) - self.fee_version = None - f = deepcopy(fee_dict) assert len(fee_dict) == 1 - for currency in fee_dict: - assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) - self.currency_symbol = currency - self.update({currency: {}}) - for version in FEE_REVISIONS: - for k in FEE_REVISIONS[version]['required']: - assert k in fee_dict[currency], "Missing required fee field: %s" % k - self[currency].update({k: f[currency].pop(k)}) - for k in FEE_REVISIONS[version]['optional']: - if k in fee_dict[currency]: - self[currency].update({k: f[currency].pop(k)}) - if not len(f): - self.fee_version = version - break - assert f[currency] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) + self.fee_version = None - self.amount = self[self.currency_symbol]['amount'] if isinstance(self[self.currency_symbol]['amount'], float) else float(self[self.currency_symbol]['amount']) + fee_to_load = deepcopy(fee_dict) + + for currency in fee_dict: + self._verify_fee(currency, fee_to_load) + + self.amount = self._get_amount() self.address = self[self.currency_symbol]['address'] + def _get_amount(self): + if isinstance(self[self.currency_symbol]['amount'], float): + return self[self.currency_symbol]['amount'] + else: + return float(self[self.currency_symbol]['amount']) -class LBRYFee(LBRYFeeFormat): + def _verify_fee(self, currency, f): + # str in case someone made a claim with a wierd fee + assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) + self.currency_symbol = currency + self.update({currency: {}}) + for version in FEE_REVISIONS: + self._load_revision(version, f) + if not f: + self.fee_version = version + break + assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) + + def _load_revision(self, version, f): + for k in FEE_REVISIONS[version]['required']: + assert k in f[self.currency_symbol], "Missing required fee field: %s" % k + self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) + for k in FEE_REVISIONS[version]['optional']: + if k in f[self.currency_symbol]: + self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) + + +class LBRYFee(LBRYFeeValidator): def __init__(self, fee_dict, rate_dict, bittrex_fee=None): - LBRYFeeFormat.__init__(self, fee_dict) + LBRYFeeValidator.__init__(self, fee_dict) self.bittrex_fee = BITTREX_FEE if bittrex_fee is None else bittrex_fee rates = deepcopy(rate_dict) @@ -102,25 +117,35 @@ class LBRYFee(LBRYFeeFormat): class Metadata(dict): def __init__(self, metadata): dict.__init__(self) - self.metaversion = None - m = deepcopy(metadata) + self.meta_version = None + metadata_to_load = deepcopy(metadata) + self._verify_sources(metadata_to_load) + self._verify_metadata(metadata_to_load) + + def _load_revision(self, version, metadata): + for k in METADATA_REVISIONS[version]['required']: + assert k in metadata, "Missing required metadata field: %s" % k + self.update({k: metadata.pop(k)}) + for k in METADATA_REVISIONS[version]['optional']: + if k == 'fee': + self['fee'] = LBRYFeeValidator(metadata.pop('fee')) + elif k in metadata: + self.update({k: metadata.pop(k)}) + + def _load_fee(self, metadata): + if 'fee' in metadata: + self['fee'] = LBRYFeeValidator(metadata.pop('fee')) + + def _verify_sources(self, metadata): assert "sources" in metadata, "No sources given" for source in metadata['sources']: assert source in SOURCE_TYPES, "Unknown source type" + def _verify_metadata(self, metadata): for version in METADATA_REVISIONS: - for k in METADATA_REVISIONS[version]['required']: - assert k in metadata, "Missing required metadata field: %s" % k - self.update({k: m.pop(k)}) - for k in METADATA_REVISIONS[version]['optional']: - if k == 'fee': - pass - elif k in metadata: - self.update({k: m.pop(k)}) - if not len(m) or m.keys() == ['fee']: - self.metaversion = version + self._load_revision(version, metadata) + if not metadata: + self.meta_version = version break - if 'fee' in m: - self['fee'] = LBRYFeeFormat(m.pop('fee')) - assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys()) + assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys()) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 3e7573a6e..1e567169d 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -379,7 +379,7 @@ class LBRYWallet(object): m = Metadata(value_dict) if 'txid' in result: d = self._save_name_metadata(name, str(result['txid']), m['sources']['lbry_sd_hash']) - d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, m.metaversion))) + d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, m.meta_version))) d.addCallback(lambda _: m) return d elif 'error' in result: diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 0056101c5..8106b1fd4 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -84,26 +84,14 @@ class GetStream(object): self.code = STREAM_STAGES[4] self.finished.callback(False) + def _convert_max_fee(self): + if isinstance(self.max_key_fee, dict): + max_fee = deepcopy(self.max_key_fee) + return LBRYFee(max_fee, {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}).to_lbc() + elif isinstance(self.max_key_fee, float): + return float(self.max_key_fee) + def start(self, stream_info, name): - self.resolved_name = name - self.stream_info = deepcopy(stream_info) - self.description = self.stream_info['description'] - self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] - - if 'fee' in self.stream_info: - self.fee = LBRYFee(self.stream_info['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) - if isinstance(self.max_key_fee, float): - if self.fee.to_lbc() > self.max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) - return defer.fail(KeyFeeAboveMaxAllowed()) - log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) - elif isinstance(self.max_key_fee, dict): - max_key = LBRYFee(deepcopy(self.max_key_fee), {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) - if self.fee.to_lbc() > max_key.to_lbc(): - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), max_key.to_lbc(), self.resolved_name)) - return defer.fail(KeyFeeAboveMaxAllowed()) - log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), max_key.to_lbc(), self.resolved_name)) - def _cause_timeout(err): log.error(err) log.debug('Forcing a timeout') @@ -128,6 +116,23 @@ class GetStream(object): download_directory=self.download_directory, file_name=self.file_name) + self.resolved_name = name + self.stream_info = deepcopy(stream_info) + self.description = self.stream_info['description'] + self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] + + if 'fee' in self.stream_info: + self.fee = LBRYFee(self.stream_info['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + max_key_fee = self._convert_max_fee() + if self.fee.to_lbc() > max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), + self.max_key_fee, + self.resolved_name)) + return defer.fail(KeyFeeAboveMaxAllowed()) + log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), + max_key_fee, + self.resolved_name)) + self.checker.start(1) self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE)) diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index ac88ffa77..643fc5d31 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -12,7 +12,7 @@ class LBRYFeeFormatTest(unittest.TestCase): 'address': None } } - fee = LBRYMetadata.LBRYFeeFormat(fee_dict) + fee = LBRYMetadata.LBRYFeeValidator(fee_dict) self.assertEqual(10, fee['USD']['amount']) @@ -68,7 +68,7 @@ class MetadataTest(unittest.TestCase): 'content-type': None, } m = LBRYMetadata.Metadata(metadata) - self.assertEquals('0.0.1', m.metaversion) + self.assertEquals('0.0.1', m.meta_version) def test_assertion_if_there_is_an_extra_field(self): metadata = { @@ -97,6 +97,6 @@ class MetadataTest(unittest.TestCase): 'ver': None } m = LBRYMetadata.Metadata(metadata) - self.assertEquals('0.0.2', m.metaversion) + self.assertEquals('0.0.2', m.meta_version) From 7d22bfffa5e98b6144f24988b5cb36e06c515fb6 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 02:30:37 -0400 Subject: [PATCH 41/73] fee tests --- tests/lbrynet/core/test_LBRYMetadata.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index 643fc5d31..e3910ed1a 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -8,12 +8,12 @@ class LBRYFeeFormatTest(unittest.TestCase): def test_fee_created_with_correct_inputs(self): fee_dict = { 'USD': { - 'amount': 10, - 'address': None + 'amount': 10.0, + 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" } } fee = LBRYMetadata.LBRYFeeValidator(fee_dict) - self.assertEqual(10, fee['USD']['amount']) + self.assertEqual(10.0, fee['USD']['amount']) class LBRYFeeTest(unittest.TestCase): @@ -28,13 +28,13 @@ class LBRYFeeTest(unittest.TestCase): def test_fee_converts_to_lbc(self): fee_dict = { 'USD': { - 'amount': 10, - 'address': None + 'amount': 10.0, + 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" } } - rates = {'BTCLBC': {'spot': 3, 'ts': 2}, 'USDBTC': {'spot': 2, 'ts': 3}} + rates = {'BTCLBC': {'spot': 3.0, 'ts': 2}, 'USDBTC': {'spot': 2.0, 'ts': 3}} fee = LBRYMetadata.LBRYFee(fee_dict, rates, 0) - self.assertEqual(60, fee.to_lbc()) + self.assertEqual(60.0, fee.to_lbc()) class MetadataTest(unittest.TestCase): From 12b2bbfa269425b23891cd4640ddb9ba624f6bf7 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 02:36:07 -0400 Subject: [PATCH 42/73] make sure claimed ver is true --- lbrynet/core/LBRYMetadata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index beed9bf12..9def7f8cd 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -5,6 +5,7 @@ import time from copy import deepcopy from googlefinance import getQuotes from lbrynet.conf import CURRENCIES +from lbrynet.core import utils import logging log = logging.getLogger(__name__) @@ -147,5 +148,7 @@ class Metadata(dict): self._load_revision(version, metadata) if not metadata: self.meta_version = version + if utils.version_is_greater_than(self.meta_version, "0.0.1"): + assert self.meta_version == self['ver'], "version mismatch" break assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys()) From e058c7101973e690b7ce2b0c9be22594fdcbdb34 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 02:46:46 -0400 Subject: [PATCH 43/73] test data --- tests/lbrynet/core/test_LBRYMetadata.py | 83 +++++++++++++++++-------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index e3910ed1a..2726f6a6d 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -45,58 +45,89 @@ class MetadataTest(unittest.TestCase): def test_assertion_if_invalid_source(self): metadata = { - 'sources': {'garbage': None} + 'license': 'Oscilloscope Laboratories', + 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'fake': 'source'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' } with self.assertRaises(AssertionError): LBRYMetadata.Metadata(metadata) def test_assertion_if_missing_v001_field(self): metadata = { - 'sources': [], + 'license': 'Oscilloscope Laboratories', + 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' } with self.assertRaises(AssertionError): LBRYMetadata.Metadata(metadata) def test_version_is_001_if_all_fields_are_present(self): metadata = { - 'sources': [], - 'title': None, - 'description': None, - 'author': None, - 'language': None, - 'license': None, - 'content-type': None, + 'license': 'Oscilloscope Laboratories', + 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' } m = LBRYMetadata.Metadata(metadata) self.assertEquals('0.0.1', m.meta_version) def test_assertion_if_there_is_an_extra_field(self): metadata = { - 'sources': [], - 'title': None, - 'description': None, - 'author': None, - 'language': None, - 'license': None, - 'content-type': None, - 'extra': None + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.2', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content-type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg', + 'MYSTERYFIELD': '?' } with self.assertRaises(AssertionError): LBRYMetadata.Metadata(metadata) def test_version_is_002_if_all_fields_are_present(self): metadata = { - 'sources': [], - 'title': None, - 'description': None, - 'author': None, - 'language': None, - 'license': None, - 'content-type': None, - 'nsfw': None, - 'ver': None + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.2', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content-type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' } m = LBRYMetadata.Metadata(metadata) self.assertEquals('0.0.2', m.meta_version) + + + From 4223298634ba1e5900ce600e622e30d7141b8331 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 02:49:35 -0400 Subject: [PATCH 44/73] test wrong metadata version --- tests/lbrynet/core/test_LBRYMetadata.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index 2726f6a6d..e474e824e 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -127,7 +127,23 @@ class MetadataTest(unittest.TestCase): m = LBRYMetadata.Metadata(metadata) self.assertEquals('0.0.2', m.meta_version) - + def test_version_claimed_is_001_but_version_is_002(self): + metadata = { + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.1', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content-type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) From 415495fc1688a41c763023ef0da4808241ae78ee Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 05:30:13 -0400 Subject: [PATCH 45/73] LBRYExchangeRateManager --- lbrynet/core/LBRYMetadata.py | 48 +----- lbrynet/core/LBRYWallet.py | 49 ------ lbrynet/lbrynet_daemon/LBRYDaemon.py | 8 +- lbrynet/lbrynet_daemon/LBRYDownloader.py | 23 +-- .../lbrynet_daemon/LBRYExchangeRateManager.py | 150 ++++++++++++++++++ 5 files changed, 170 insertions(+), 108 deletions(-) create mode 100644 lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 9def7f8cd..2bf31c9c7 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -32,6 +32,7 @@ class LBRYFeeValidator(dict): dict.__init__(self) assert len(fee_dict) == 1 self.fee_version = None + self.currency_symbol = None fee_to_load = deepcopy(fee_dict) @@ -68,53 +69,6 @@ class LBRYFeeValidator(dict): self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) -class LBRYFee(LBRYFeeValidator): - def __init__(self, fee_dict, rate_dict, bittrex_fee=None): - LBRYFeeValidator.__init__(self, fee_dict) - self.bittrex_fee = BITTREX_FEE if bittrex_fee is None else bittrex_fee - rates = deepcopy(rate_dict) - - assert 'BTCLBC' in rates and 'USDBTC' in rates - for fx in rate_dict: - assert int(time.time()) - int(rates[fx]['ts']) < 3600, "%s quote is out of date" % fx - self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']} - self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']} - - def to_lbc(self): - r = None - if self.currency_symbol == "LBC": - r = round(float(self.amount), 5) - elif self.currency_symbol == "BTC": - r = round(float(self._btc_to_lbc(self.amount)), 5) - elif self.currency_symbol == "USD": - r = round(float(self._btc_to_lbc(self._usd_to_btc(self.amount))), 5) - assert r is not None - return r - - def to_usd(self): - r = None - if self.currency_symbol == "USD": - r = round(float(self.amount), 5) - elif self.currency_symbol == "BTC": - r = round(float(self._btc_to_usd(self.amount)), 5) - elif self.currency_symbol == "LBC": - r = round(float(self._btc_to_usd(self._lbc_to_btc(self.amount))), 5) - assert r is not None - return r - - def _usd_to_btc(self, usd): - return self._USDBTC['spot'] * float(usd) - - def _btc_to_usd(self, btc): - return float(btc) / self._USDBTC['spot'] - - def _btc_to_lbc(self, btc): - return float(btc) * self._BTCLBC['spot'] / (1.0 - self.bittrex_fee) - - def _lbc_to_btc(self, lbc): - return self._BTCLBC['spot'] / float(lbc) - - class Metadata(dict): def __init__(self, metadata): dict.__init__(self) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 1e567169d..7d7ab8db7 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -82,50 +82,6 @@ class LBRYWallet(object): self._batch_count = 20 self._first_run = self._FIRST_RUN_UNKNOWN - self._USDBTC = None - self._BTCLBC = None - self._exchange_rate_updater = task.LoopingCall(self._update_exchange_rates) - - def _usd_to_btc(self): - if self._USDBTC is not None: - if int(time.time()) - int(self._USDBTC['ts']) < 600: - log.info("USDBTC quote is new enough") - return defer.succeed({}) - - log.info("Getting new USDBTC quote") - x = float(getQuotes('CURRENCY:USDBTC')[0]['LastTradePrice']) - return defer.succeed({'USDBTC': {'spot': x, 'ts': int(time.time())}}) - - def _btc_to_lbc(self): - if self._BTCLBC is not None: - if int(time.time()) - int(self._BTCLBC['ts']) < 600: - log.info("BTCLBC quote is new enough") - return defer.succeed({}) - - log.info("Getting new BTCLBC quote") - r = requests.get("https://bittrex.com/api/v1.1/public/getmarkethistory", {'market': 'BTC-LBC', 'count': 50}) - trades = json.loads(r.text)['result'] - vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades]) - x = (1.0 / float(vwap)) / 0.99975 - - return defer.succeed({'BTCLBC': {'spot': x, 'ts': int(time.time())}}) - - def _set_exchange_rates(self, rates): - if 'USDBTC' in rates: - assert int(time.time()) - int(rates['USDBTC']['ts']) < 3600, "new USDBTC quote is too old" - self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']} - log.info("Updated USDBTC rate: %s" % json.dumps(self._USDBTC)) - if 'BTCLBC' in rates: - assert int(time.time()) - int(rates['BTCLBC']['ts']) < 3600, "new BTCLBC quote is too old" - self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']} - log.info("Updated BTCLBC rate: %s" % json.dumps(self._BTCLBC)) - - def _update_exchange_rates(self): - d = self._usd_to_btc() - d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) - d.addCallback(lambda _: self._btc_to_lbc()) - d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) - def start(self): def start_manage(): @@ -133,8 +89,6 @@ class LBRYWallet(object): self.manage() return True - self._exchange_rate_updater.start(1800) - d = self._open_db() d.addCallback(lambda _: self._start()) d.addCallback(lambda _: start_manage()) @@ -147,9 +101,6 @@ class LBRYWallet(object): def stop(self): self.stopped = True - - if self._exchange_rate_updater.running: - self._exchange_rate_updater.stop() # If self.next_manage_call is None, then manage is currently running or else # start has not been called, so set stopped and do nothing else. diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 601c6ae0f..89b07e504 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -41,6 +41,7 @@ from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifi from lbrynet.lbrynet_daemon.LBRYUIManager import LBRYUIManager from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher +from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.core import utils from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings @@ -157,6 +158,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.current_db_revision = 1 self.run_server = True self.session = None + self.exchange_rate_manager = ExchangeRateManager() self.waiting_on = {} self.streams = {} self.pending_claims = {} @@ -484,6 +486,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) self.connection_problem_checker.start(1) + self.exchange_rate_manager.start() + if host_ui: self.lbry_ui_manager.update_checker.start(1800, now=False) @@ -1122,8 +1126,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet, - self.lbry_file_manager, max_key_fee=self.max_key_fee, - data_rate=self.data_rate, timeout=timeout, + self.lbry_file_manager, self.exchange_rate_manager, + max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout, download_directory=download_directory, file_name=file_name) d = self.streams[name].start(stream_info, name) if wait_for_write: diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 8106b1fd4..62d9cd33d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -12,7 +12,7 @@ from twisted.internet.task import LoopingCall from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob -from lbrynet.core.LBRYMetadata import Metadata, LBRYFee +from lbrynet.core.LBRYMetadata import Metadata, LBRYFeeValidator from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME @@ -42,8 +42,8 @@ log = logging.getLogger(__name__) class GetStream(object): - def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, data_rate=0.5, - timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None): + def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager, + max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None): self.wallet = wallet self.resolved_name = None self.description = None @@ -52,6 +52,7 @@ class GetStream(object): self.name = None self.file_name = file_name self.session = session + self.exchange_rate_manager = exchange_rate_manager self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager) self.lbry_file_manager = lbry_file_manager self.sd_identifier = sd_identifier @@ -86,8 +87,10 @@ class GetStream(object): def _convert_max_fee(self): if isinstance(self.max_key_fee, dict): - max_fee = deepcopy(self.max_key_fee) - return LBRYFee(max_fee, {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}).to_lbc() + max_fee = LBRYFeeValidator(self.max_key_fee) + if max_fee.currency_symbol == "LBC": + return max_fee.amount + return self.exchange_rate_manager.to_lbc(self.fee).amount elif isinstance(self.max_key_fee, float): return float(self.max_key_fee) @@ -122,14 +125,14 @@ class GetStream(object): self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] if 'fee' in self.stream_info: - self.fee = LBRYFee(self.stream_info['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + self.fee = LBRYFeeValidator(self.stream_info['fee']) max_key_fee = self._convert_max_fee() - if self.fee.to_lbc() > max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), + if self.exchange_rate_manager.to_lbc(self.fee).amount > max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount, self.max_key_fee, self.resolved_name)) return defer.fail(KeyFeeAboveMaxAllowed()) - log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), + log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee), max_key_fee, self.resolved_name)) @@ -149,7 +152,7 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): if self.fee is not None: - fee_lbc = float(self.fee.to_lbc()) + fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) if reserved_points is None: return defer.fail(InsufficientFundsError()) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py new file mode 100644 index 000000000..2db0d526a --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -0,0 +1,150 @@ +import time +import requests +import logging +import json +import googlefinance +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall + +from lbrynet.core.LBRYMetadata import LBRYFeeValidator + +log = logging.getLogger(__name__) + +CURRENCY_PAIRS = ["USDBTC", "BTCLBC"] +BITTREX_FEE = 0.0025 +COINBASE_FEE = 0.0 #add fee + + +class ExchangeRate(object): + def __init__(self, market, spot, ts): + assert int(time.time()) - ts < 600 + self.currency_pair = (market[0:3], market[3:6]) + self.spot = spot + self.ts = ts + + def as_dict(self): + return {'spot': self.spot, 'ts': self.ts} + + +class MarketFeed(object): + def __init__(self, market, name, url, params, fee): + self.market = market + self.name = name + self.url = url + self.params = params + self.fee = fee + self.rate = None + self._updater = LoopingCall(self._update_price) + + def _make_request(self): + r = requests.get(self.url, self.params) + return r.text + + def _handle_response(self, response): + return NotImplementedError + + def _subtract_fee(self, from_amount): + return defer.succeed(from_amount / (1.0 - self.fee)) + + def _save_price(self, price): + log.info("Saving price update %f for %s" % (price, self.market)) + self.rate = ExchangeRate(self.market, price, int(time.time())) + + def _update_price(self): + d = defer.succeed(self._make_request()) + d.addCallback(self._handle_response) + d.addCallback(self._subtract_fee) + d.addCallback(self._save_price) + + def start(self): + if not self._updater.running: + self._updater.start(15) + + def stop(self): + if self._updater.running: + self._updater.stop() + + +class BittrexFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "BTCLBC", + "Bittrex", + "https://bittrex.com/api/v1.1/public/getmarkethistory", + {'market': 'BTC-LBC', 'count': 50}, + BITTREX_FEE + ) + + def _handle_response(self, response): + trades = json.loads(response)['result'] + vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades]) + return defer.succeed(float(1.0 / vwap)) + + +class GoogleBTCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "USDBTC", + "Coinbase via Google finance", + None, + None, + COINBASE_FEE + ) + + def _make_request(self): + return googlefinance.getQuotes('CURRENCY:USDBTC')[0] + + def _handle_response(self, response): + return float(response['LastTradePrice']) + + +def get_default_market_feed(currency_pair): + currencies = None + if isinstance(currency_pair, str): + currencies = (currency_pair[0:3], currency_pair[3:6]) + elif isinstance(currency_pair, tuple): + currencies = currency_pair + assert currencies is not None + + if currencies == ("USD", "BTC"): + return GoogleBTCFeed() + elif currencies == ("BTC", "LBC"): + return BittrexFeed() + + +class ExchangeRateManager(object): + def __init__(self): + reactor.addSystemEventTrigger('before', 'shutdown', self.stop) + self.market_feeds = [get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS] + + def start(self): + log.info("Starting exchange rate manager") + for feed in self.market_feeds: + feed.start() + + def stop(self): + log.info("Stopping exchange rate manager") + for source in self.market_feeds: + source.stop() + + def convert_currency(self, from_currency, to_currency, amount): + log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) + for market in self.market_feeds: + if market.rate.currency_pair == (from_currency, to_currency): + return amount * market.rate.spot + for market in self.market_feeds: + if market.rate.currency_pair[0] == from_currency: + return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) + + def fee_dict(self): + return {market: market.rate.as_dict() for market in self.market_feeds} + + def to_lbc(self, fee): + return LBRYFeeValidator({fee.currency_symbol: + { + 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee.amount), + 'address': fee.address + } + }) From 52d954c64ecdaf3a740eb87202797f065422b6b5 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 05:34:20 -0400 Subject: [PATCH 46/73] don't spam the market apis --- lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index 2db0d526a..6fd470622 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -58,7 +58,7 @@ class MarketFeed(object): def start(self): if not self._updater.running: - self._updater.start(15) + self._updater.start(300) def stop(self): if self._updater.running: From 5441379f380473b407994f5e308f042fd171410e Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 05:49:31 -0400 Subject: [PATCH 47/73] fix LBRYExchangeRateManager test --- .../lbrynet_daemon/LBRYExchangeRateManager.py | 48 +++++++++++++++++++ tests/lbrynet/core/test_LBRYMetadata.py | 5 +- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index 6fd470622..b8a3538d0 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -148,3 +148,51 @@ class ExchangeRateManager(object): 'address': fee.address } }) + + +class DummyBTCLBCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "BTCLBC", + "market name", + "derp.com", + None, + 0.0 + ) + + +class DummyUSDBTCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "USDBTC", + "market name", + "derp.com", + None, + 0.0 + ) + + +class DummyExchangeRateManager(object): + def __init__(self, rates): + self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()] + for feed in self.market_feeds: + feed.rate = rates[feed.market] + + def convert_currency(self, from_currency, to_currency, amount): + log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) + for market in self.market_feeds: + if market.rate.currency_pair == (from_currency, to_currency): + return amount * market.rate.spot + for market in self.market_feeds: + if market.rate.currency_pair[0] == from_currency: + return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) + + def to_lbc(self, fee): + return LBRYFeeValidator({fee.currency_symbol: + { + 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee.amount), + 'address': fee.address + } + }) \ No newline at end of file diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index e474e824e..6f124e38d 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -1,5 +1,6 @@ import mock from lbrynet.core import LBRYMetadata +from lbrynet.lbrynet_daemon import LBRYExchangeRateManager from twisted.trial import unittest @@ -33,8 +34,8 @@ class LBRYFeeTest(unittest.TestCase): } } rates = {'BTCLBC': {'spot': 3.0, 'ts': 2}, 'USDBTC': {'spot': 2.0, 'ts': 3}} - fee = LBRYMetadata.LBRYFee(fee_dict, rates, 0) - self.assertEqual(60.0, fee.to_lbc()) + manager = LBRYExchangeRateManager.DummyExchangeRateManager(rates) + self.assertEqual(60.0, manager.to_lbc(fee_dict).amount) class MetadataTest(unittest.TestCase): From 6b1d11a04d85e75705a3c06188348b0102074c35 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 05:53:31 -0400 Subject: [PATCH 48/73] fix rate in test --- lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index b8a3538d0..a7bf37ef8 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -178,7 +178,7 @@ class DummyExchangeRateManager(object): def __init__(self, rates): self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()] for feed in self.market_feeds: - feed.rate = rates[feed.market] + feed.rate = ExchangeRate(feed, rates[feed]['spot'], rates[feed]['ts']) def convert_currency(self, from_currency, to_currency, amount): log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) From 413dab8094ce0bb0909b1ece4465bd3eedb6e179 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 05:59:10 -0400 Subject: [PATCH 49/73] whoops --- lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index a7bf37ef8..e0656cfae 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -178,7 +178,7 @@ class DummyExchangeRateManager(object): def __init__(self, rates): self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()] for feed in self.market_feeds: - feed.rate = ExchangeRate(feed, rates[feed]['spot'], rates[feed]['ts']) + feed.rate = ExchangeRate(feed, rates[feed.market]['spot'], rates[feed.market]['ts']) def convert_currency(self, from_currency, to_currency, amount): log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) From 52c20f02c7d301577488ee75ef8d780508ba3c5a Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 06:03:47 -0400 Subject: [PATCH 50/73] one more --- lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index e0656cfae..3bf6a30ab 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -178,7 +178,7 @@ class DummyExchangeRateManager(object): def __init__(self, rates): self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()] for feed in self.market_feeds: - feed.rate = ExchangeRate(feed, rates[feed.market]['spot'], rates[feed.market]['ts']) + feed.rate = ExchangeRate(feed.market, rates[feed.market]['spot'], rates[feed.market]['ts']) def convert_currency(self, from_currency, to_currency, amount): log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) From 4387025c489526f80beb30dda7853d727604af41 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 06:07:33 -0400 Subject: [PATCH 51/73] another still --- lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index 3bf6a30ab..cc924ab7e 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -189,7 +189,8 @@ class DummyExchangeRateManager(object): if market.rate.currency_pair[0] == from_currency: return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) - def to_lbc(self, fee): + def to_lbc(self, f): + fee = LBRYFeeValidator(f) return LBRYFeeValidator({fee.currency_symbol: { 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee.amount), From 15d276b0bc37e14f160b9e358e8f56b05fd449fc Mon Sep 17 00:00:00 2001 From: Job Evers Date: Thu, 28 Jul 2016 11:43:20 -0500 Subject: [PATCH 52/73] fix case where to and from currencies are the same --- lbrynet/core/LBRYMetadata.py | 11 ++++++++--- lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 2bf31c9c7..9a0ea49c4 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -43,10 +43,15 @@ class LBRYFeeValidator(dict): self.address = self[self.currency_symbol]['address'] def _get_amount(self): - if isinstance(self[self.currency_symbol]['amount'], float): - return self[self.currency_symbol]['amount'] + amt = self[self.currency_symbol]['amount'] + if isinstance(amt, float): + return amt else: - return float(self[self.currency_symbol]['amount']) + try: + return float(amt) + except TypeError: + log.error('Failed to convert %s to float', amt) + raise def _verify_fee(self, currency, f): # str in case someone made a claim with a wierd fee diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index cc924ab7e..b578a2983 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -131,12 +131,15 @@ class ExchangeRateManager(object): def convert_currency(self, from_currency, to_currency, amount): log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) + if from_currency == to_currency: + return amount for market in self.market_feeds: if market.rate.currency_pair == (from_currency, to_currency): return amount * market.rate.spot for market in self.market_feeds: if market.rate.currency_pair[0] == from_currency: return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) + raise Exception('Unable to convert {} from {} to {}'.format(amount, from_current, to_currency)) def fee_dict(self): return {market: market.rate.as_dict() for market in self.market_feeds} From 39a7f37f3584a357b117c5905e92e539e42a3642 Mon Sep 17 00:00:00 2001 From: Job Evers Date: Thu, 28 Jul 2016 11:43:47 -0500 Subject: [PATCH 53/73] muck with pending_claims... still not sure this is correct --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 89b07e504..adbbe7ff8 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -608,20 +608,25 @@ class LBRYDaemon(jsonrpc.JSONRPC): return txid def _check_pending_claims(self): - def _start_file(name): + # TODO: this was blatantly copied from jsonrpc_start_lbry_file. Be DRY. + def _start_file(f): + d = self.lbry_file_manager.toggle_lbry_file_running(f) + return defer.succeed("Started LBRY file") + + def _get_and_start_file(name): d = defer.succeed(self.pending_claims.pop(name)) d.addCallback(lambda _: self._get_lbry_file("name", name, return_json=False)) d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running") def _process_lbry_file(name, lbry_file): # lbry_file is an instance of ManagedLBRYFileDownloader or None + # TODO: check for sd_hash in addition to txid ready_to_start = ( lbry_file and - self.pending_claims[name] == lbry_file.txid and - not isinstance(lbry_file['metadata'], str) + self.pending_claims[name] == lbry_file.txid ) if ready_to_start: - _start_file(name) + _get_and_start_file(name, lbry_file) else: re_add_to_pending_claims(name) From 67c5d15f51fc3944bfb51c96b073d6db5a49a7db Mon Sep 17 00:00:00 2001 From: Job Evers Date: Thu, 28 Jul 2016 11:46:33 -0500 Subject: [PATCH 54/73] stupid typo --- lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index b578a2983..ca7cfdbc4 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -139,7 +139,7 @@ class ExchangeRateManager(object): for market in self.market_feeds: if market.rate.currency_pair[0] == from_currency: return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) - raise Exception('Unable to convert {} from {} to {}'.format(amount, from_current, to_currency)) + raise Exception('Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency)) def fee_dict(self): return {market: market.rate.as_dict() for market in self.market_feeds} From 295e427abe686b1b0f1b02f9c6e42b4b6ae1f7ee Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 28 Jul 2016 09:42:16 -0500 Subject: [PATCH 55/73] bug fix: check if fee is in metadata before loading --- lbrynet/core/LBRYMetadata.py | 2 +- tests/lbrynet/core/test_LBRYMetadata.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 9a0ea49c4..7c9450baf 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -89,7 +89,7 @@ class Metadata(dict): self.update({k: metadata.pop(k)}) for k in METADATA_REVISIONS[version]['optional']: if k == 'fee': - self['fee'] = LBRYFeeValidator(metadata.pop('fee')) + self._load_fee(metadata) elif k in metadata: self.update({k: metadata.pop(k)}) diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index 6f124e38d..6f3023a1e 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -44,6 +44,21 @@ class MetadataTest(unittest.TestCase): with self.assertRaises(AssertionError): LBRYMetadata.Metadata(metadata) + def test_metadata_works_without_fee(self): + metadata = { + 'license': 'Oscilloscope Laboratories', + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + } + m = LBRYMetadata.Metadata(metadata) + self.assertFalse('key' in m) + def test_assertion_if_invalid_source(self): metadata = { 'license': 'Oscilloscope Laboratories', From 881168dc7a9f43b21b21c442722b1f6e8f20df06 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 13:16:33 -0400 Subject: [PATCH 56/73] fix too many args --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index adbbe7ff8..914d6360b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -618,6 +618,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._get_lbry_file("name", name, return_json=False)) d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running") + def re_add_to_pending_claims(name): + txid = self.pending_claims.pop(name) + self._add_to_pending_claims(name, txid) + def _process_lbry_file(name, lbry_file): # lbry_file is an instance of ManagedLBRYFileDownloader or None # TODO: check for sd_hash in addition to txid @@ -626,14 +630,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.pending_claims[name] == lbry_file.txid ) if ready_to_start: - _get_and_start_file(name, lbry_file) + _get_and_start_file(name) else: re_add_to_pending_claims(name) - def re_add_to_pending_claims(name): - txid = self.pending_claims.pop(name) - self._add_to_pending_claims(name, txid) - for name in self.pending_claims: log.info("Checking if new claim for lbry://%s is confirmed" % name) d = self._resolve_name(name, force_refresh=True) From 5dd68a7ed4c26fe99c621ec3eb0d0522dd49aa45 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 14:55:17 -0400 Subject: [PATCH 57/73] add get_claim_info, clean up _get_stream_info_from_value --- lbrynet/core/LBRYWallet.py | 61 +++++++++++++++++++++------- lbrynet/lbrynet_daemon/LBRYDaemon.py | 11 +++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 7d7ab8db7..60696624b 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -320,26 +320,57 @@ class LBRYWallet(object): return d def _get_stream_info_from_value(self, result, name): - if 'value' in result: - value = result['value'] + def _check_result_fields(r): + for k in ['value', 'txid', 'n', 'height', 'amount']: + assert k in r, "getvalueforname response missing field %s" % k - try: - value_dict = json.loads(value) - except (ValueError, TypeError): - return Failure(InvalidStreamInfoError(name)) - m = Metadata(value_dict) - if 'txid' in result: - d = self._save_name_metadata(name, str(result['txid']), m['sources']['lbry_sd_hash']) - d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, m.meta_version))) - d.addCallback(lambda _: m) - return d - elif 'error' in result: + if 'error' in result: log.warning("Got an error looking up a name: %s", result['error']) return Failure(UnknownNameError(name)) - else: - log.warning("Got an error looking up a name: %s", json.dumps(result)) + + _check_result_fields(result) + + try: + metadata = Metadata(json.loads(result['value'])) + except (ValueError, TypeError): + return Failure(InvalidStreamInfoError(name)) + + d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash']) + d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version))) + d.addCallback(lambda _: metadata) + return d + + def _get_claim_info(self, result, name): + def _check_result_fields(r): + for k in ['value', 'txid', 'n', 'height', 'amount']: + assert k in r, "getvalueforname response missing field %s" % k + + def _build_response(m, result): + result['value'] = m + return result + + if 'error' in result: + log.warning("Got an error looking up a name: %s", result['error']) return Failure(UnknownNameError(name)) + _check_result_fields(result) + + try: + metadata = Metadata(json.loads(result['value'])) + except (ValueError, TypeError): + return Failure(InvalidStreamInfoError(name)) + + d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash']) + d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version))) + d.addCallback(lambda _: _build_response(metadata, result)) + return d + + def get_claim_info(self, name): + d = self._get_value_for_name(name) + d.addCallback(lambda r: self._get_claim_info(r, name)) + return d + + def claim_name(self, name, bid, m): metadata = Metadata(m) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 914d6360b..b41f4eecf 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1718,6 +1718,17 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d + def jsonrpc_get_claim_info(self, p): + def _convert_amount_to_float(r): + r['amount'] = float(r['amount']) / 10**8 + return r + + name = p['name'] + d = self.session.wallet.get_claim_info(name) + d.addCallback(_convert_amount_to_float) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + return d + def jsonrpc_get(self, p): """ Download stream from a LBRY uri From de4ddd578f0355713e1848d2228fab714835d7ca Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 15:24:20 -0400 Subject: [PATCH 58/73] improve readability of metadata and fee requirements --- lbrynet/core/LBRYMetadata.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 7c9450baf..b71d0723e 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -12,19 +12,27 @@ log = logging.getLogger(__name__) BITTREX_FEE = 0.0025 +# Metadata version SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] -# v0.0.1 metadata -METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}} -# v0.0.2 metadata additions -METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']} -CURRENT_METADATA_VERSION = '0.0.2' +MV001 = "0.0.1" +MV002 = "0.0.2" +CURRENT_METADATA_VERSION = MV002 -# v0.0.1 fee -FEE_REVISIONS = {'0.0.1': {'required': ['amount', 'address'], 'optional': []}} -CURRENT_FEE_REVISION = '0.0.1' +METADATA_REVISIONS = {} +METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS} +METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']} + +# Fee version +BASE_FEE_FIELDS = ['amount', 'address'] + +FV001 = "0.0.1" +CURRENT_FEE_REVISION = FV001 + +FEE_REVISIONS = {} +FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []} class LBRYFeeValidator(dict): From 419aa1d78f01e896609537070761cdd036a8609b Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 16:12:20 -0400 Subject: [PATCH 59/73] fix est data cost / search --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 40 +++++++++++++++---- .../lbrynet_daemon/LBRYExchangeRateManager.py | 33 ++++++++++----- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index b41f4eecf..f32b5c9aa 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -34,7 +34,6 @@ from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerF from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory from lbrynet.core.Error import UnknownNameError, InsufficientFundsError -from lbrynet.core.LBRYMetadata import Metadata from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier @@ -1228,7 +1227,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _add_key_fee(data_cost): d = self._resolve_name(name) - d.addCallback(lambda info: data_cost if 'fee' not in info else data_cost + info['fee']['LBC']['amount']) + d.addCallback(lambda info: self.exchange_rate_manager.to_lbc(info.get('fee', None))) + d.addCallback(lambda fee: data_cost if fee is None else data_cost + fee.amount) return d d = self._resolve_name(name) @@ -1238,8 +1238,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) d.addCallback(lambda metadata: metadata.validator.info_to_show()) d.addCallback(lambda info: int(dict(info)['stream_size']) / 1000000 * self.data_rate) - d.addCallback(_add_key_fee) - d.addErrback(lambda _: _add_key_fee(0.0)) + d.addCallbacks(_add_key_fee, lambda _: _add_key_fee(0.0)) reactor.callLater(self.search_timeout, _check_est, d, name) return d @@ -1405,6 +1404,11 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) + def _search(self, search): + d = self.session.wallet.get_nametrie() + d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) + return d + def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) @@ -1719,6 +1723,15 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d def jsonrpc_get_claim_info(self, p): + """ + Resolve claim info from a LBRY uri + + Args: + 'name': name to look up, string, do not include lbry:// prefix + Returns: + txid, amount, value, n, height + """ + def _convert_amount_to_float(r): r['amount'] = float(r['amount']) / 10**8 return r @@ -1839,6 +1852,21 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + def jsonrpc_get_est_cost(self, p): + """ + Get estimated cost for a lbry uri + + Args: + 'name': lbry uri + Returns: + estimated cost + """ + + name = p['name'] + d = self._get_est_cost(name) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + return d + def jsonrpc_search_nametrie(self, p): """ Search the nametrie for claims beginning with search (yes, this is a dumb search, it'll be made better) @@ -1890,8 +1918,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info('Search nametrie: ' + search) - d = self.session.wallet.get_nametrie() - d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) + d = self._search(search) d.addCallback(lambda claims: claims[:self.max_search_results]) d.addCallback(resolve_claims) d.addCallback(_clean) @@ -2337,7 +2364,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): # No easy way to reveal specific files on Linux, so just open the containing directory d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.dirname(path)]) - d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index ca7cfdbc4..52f7c3661 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -145,10 +145,17 @@ class ExchangeRateManager(object): return {market: market.rate.as_dict() for market in self.market_feeds} def to_lbc(self, fee): - return LBRYFeeValidator({fee.currency_symbol: + if fee is None: + return None + if not isinstance(fee, LBRYFeeValidator): + fee_in = LBRYFeeValidator(fee) + else: + fee_in = fee + + return LBRYFeeValidator({fee_in.currency_symbol: { - 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee.amount), - 'address': fee.address + 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee_in.amount), + 'address': fee_in.address } }) @@ -192,11 +199,17 @@ class DummyExchangeRateManager(object): if market.rate.currency_pair[0] == from_currency: return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) - def to_lbc(self, f): - fee = LBRYFeeValidator(f) - return LBRYFeeValidator({fee.currency_symbol: - { - 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee.amount), - 'address': fee.address - } + def to_lbc(self, fee): + if fee is None: + return None + if not isinstance(fee, LBRYFeeValidator): + fee_in = LBRYFeeValidator(fee) + else: + fee_in = fee + + return LBRYFeeValidator({fee_in.currency_symbol: + { + 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee_in.amount), + 'address': fee_in.address + } }) \ No newline at end of file From 2e3342d20d9bd8aca93e7d9148f2275b6293c2f3 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 16:27:16 -0400 Subject: [PATCH 60/73] fix test --- .../lbrynet_daemon/LBRYExchangeRateManager.py | 12 ++---- .../core/test_LBRYExchangeRateManager.py | 38 +++++++++++++++++++ tests/lbrynet/core/test_LBRYMetadata.py | 36 ------------------ 3 files changed, 42 insertions(+), 44 deletions(-) create mode 100644 tests/lbrynet/core/test_LBRYExchangeRateManager.py diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index 52f7c3661..52424cb3e 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -147,10 +147,8 @@ class ExchangeRateManager(object): def to_lbc(self, fee): if fee is None: return None - if not isinstance(fee, LBRYFeeValidator): - fee_in = LBRYFeeValidator(fee) - else: - fee_in = fee + + fee_in = LBRYFeeValidator(fee) return LBRYFeeValidator({fee_in.currency_symbol: { @@ -202,10 +200,8 @@ class DummyExchangeRateManager(object): def to_lbc(self, fee): if fee is None: return None - if not isinstance(fee, LBRYFeeValidator): - fee_in = LBRYFeeValidator(fee) - else: - fee_in = fee + + fee_in = LBRYFeeValidator(fee) return LBRYFeeValidator({fee_in.currency_symbol: { diff --git a/tests/lbrynet/core/test_LBRYExchangeRateManager.py b/tests/lbrynet/core/test_LBRYExchangeRateManager.py new file mode 100644 index 000000000..2a6457536 --- /dev/null +++ b/tests/lbrynet/core/test_LBRYExchangeRateManager.py @@ -0,0 +1,38 @@ +import mock +from lbrynet.core import LBRYMetadata +from lbrynet.lbrynet_daemon import LBRYExchangeRateManager + +from twisted.trial import unittest + + +class LBRYFeeFormatTest(unittest.TestCase): + def test_fee_created_with_correct_inputs(self): + fee_dict = { + 'USD': { + 'amount': 10.0, + 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" + } + } + fee = LBRYMetadata.LBRYFeeValidator(fee_dict) + self.assertEqual(10.0, fee['USD']['amount']) + + +class LBRYFeeTest(unittest.TestCase): + def setUp(self): + self.patcher = mock.patch('time.time') + self.time = self.patcher.start() + self.time.return_value = 0 + + def tearDown(self): + self.time.stop() + + def test_fee_converts_to_lbc(self): + fee_dict = { + 'USD': { + 'amount': 10.0, + 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" + } + } + rates = {'BTCLBC': {'spot': 3.0, 'ts': 2}, 'USDBTC': {'spot': 2.0, 'ts': 3}} + manager = LBRYExchangeRateManager.DummyExchangeRateManager(rates) + self.assertEqual(60.0, manager.to_lbc(fee_dict).amount) \ No newline at end of file diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index 6f3023a1e..e5c3255b3 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -1,43 +1,7 @@ -import mock from lbrynet.core import LBRYMetadata -from lbrynet.lbrynet_daemon import LBRYExchangeRateManager - from twisted.trial import unittest -class LBRYFeeFormatTest(unittest.TestCase): - def test_fee_created_with_correct_inputs(self): - fee_dict = { - 'USD': { - 'amount': 10.0, - 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" - } - } - fee = LBRYMetadata.LBRYFeeValidator(fee_dict) - self.assertEqual(10.0, fee['USD']['amount']) - - -class LBRYFeeTest(unittest.TestCase): - def setUp(self): - self.patcher = mock.patch('time.time') - self.time = self.patcher.start() - self.time.return_value = 0 - - def tearDown(self): - self.time.stop() - - def test_fee_converts_to_lbc(self): - fee_dict = { - 'USD': { - 'amount': 10.0, - 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" - } - } - rates = {'BTCLBC': {'spot': 3.0, 'ts': 2}, 'USDBTC': {'spot': 2.0, 'ts': 3}} - manager = LBRYExchangeRateManager.DummyExchangeRateManager(rates) - self.assertEqual(60.0, manager.to_lbc(fee_dict).amount) - - class MetadataTest(unittest.TestCase): def test_assertion_if_source_is_missing(self): metadata = {} From 52024986eea2831d1064e57c3808a7286199fdc5 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 16:32:59 -0400 Subject: [PATCH 61/73] derp --- .../lbrynet_daemon/LBRYExchangeRateManager.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index 52424cb3e..0af938954 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -147,12 +147,14 @@ class ExchangeRateManager(object): def to_lbc(self, fee): if fee is None: return None - - fee_in = LBRYFeeValidator(fee) + if not isinstance(fee, LBRYFeeValidator): + fee_in = LBRYFeeValidator(fee) + else: + fee_in = fee return LBRYFeeValidator({fee_in.currency_symbol: { - 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee_in.amount), + 'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount), 'address': fee_in.address } }) @@ -200,12 +202,14 @@ class DummyExchangeRateManager(object): def to_lbc(self, fee): if fee is None: return None - - fee_in = LBRYFeeValidator(fee) + if not isinstance(fee, LBRYFeeValidator): + fee_in = LBRYFeeValidator(fee) + else: + fee_in = fee return LBRYFeeValidator({fee_in.currency_symbol: { - 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee_in.amount), + 'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount), 'address': fee_in.address } }) \ No newline at end of file From 68216768539f9504e5bbf8f5f61fe539ca684eda Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 18:12:54 -0400 Subject: [PATCH 62/73] handle decimals in jsonrpc return --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index f32b5c9aa..a89bee25c 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -417,6 +417,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): return server.NOT_DONE_YET def _cbRender(self, result, request, id, version): + def default_decimal(obj): + if isinstance(obj, Decimal): + return float(obj) + if isinstance(result, Handler): result = result.result @@ -428,7 +432,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): result = (result,) # Convert the result (python) to JSON-RPC try: - s = jsonrpclib.dumps(result, version=version) + s = jsonrpclib.dumps(result, version=version, default=default_decimal) except: f = jsonrpclib.Fault(self.FAILURE, "can't serialize output") s = jsonrpclib.dumps(f, version=version) From b83dce2cb32260017045794cd3b5403fe1fc99ca Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 18:48:29 -0400 Subject: [PATCH 63/73] no tricky characters --- lbrynet/core/Error.py | 8 ++++++++ lbrynet/core/LBRYMetadata.py | 7 +++++++ lbrynet/lbrynet_daemon/LBRYDaemon.py | 14 +++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index 631c5685c..8146dc169 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -34,6 +34,14 @@ class UnknownNameError(Exception): return repr(self.name) +class InvalidNameError(Exception): + def __init__(self, name): + self.name = name + + def __str__(self): + return repr(self.name) + + class UnknownStreamTypeError(Exception): def __init__(self, stream_type): self.stream_type = stream_type diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index b71d0723e..656b55c99 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -14,6 +14,7 @@ BITTREX_FEE = 0.0025 # Metadata version SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] +NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-' BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] @@ -35,6 +36,12 @@ FEE_REVISIONS = {} FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []} +def verify_name_characters(name): + for c in name: + assert c in NAME_ALLOWED_CHARSET, "Invalid character" + return True + + class LBRYFeeValidator(dict): def __init__(self, fee_dict): dict.__init__(self) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index a89bee25c..b18833056 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -33,7 +33,7 @@ from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory -from lbrynet.core.Error import UnknownNameError, InsufficientFundsError +from lbrynet.core.Error import UnknownNameError, InsufficientFundsError, InvalidNameError from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier @@ -42,6 +42,7 @@ from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.core import utils +from lbrynet.core.LBRYMetadata import verify_name_characters from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ @@ -1165,6 +1166,12 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _resolve_name(self, name, force_refresh=False): + try: + verify_name_characters(name) + except: + log.error("Bad name") + return defer.fail(InvalidNameError("Bad name")) + def _cache_stream_info(stream_info): def _add_txid(txid): self.name_cache[name]['txid'] = txid @@ -1975,6 +1982,11 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ name = p['name'] + try: + verify_name_characters(name) + except: + log.error("Bad name") + return defer.fail(InvalidNameError("Bad name")) bid = p['bid'] file_path = p['file_path'] metadata = p['metadata'] From 33e7e4bee385086b82f94e347b9aa2d163e104b0 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 21:25:37 -0400 Subject: [PATCH 64/73] fancy search --- lbrynet/conf.py | 1 + lbrynet/lbrynet_daemon/LBRYDaemon.py | 60 ++++++++++++++-------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 85a9f6836..9576cc3b4 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -24,6 +24,7 @@ KNOWN_DHT_NODES = [('104.236.42.182', 4000), POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424' #POINTTRADER_SERVER = 'http://127.0.0.1:2424' +SEARCH_SERVER = "http://45.63.4.203:50005" LOG_FILE_NAME = "lbrynet.log" LOG_POST_URL = "https://lbry.io/log-upload" diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index b18833056..09ff51351 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -25,7 +25,7 @@ from twisted.internet import defer, threads, error, reactor from twisted.internet.task import LoopingCall from txjsonrpc import jsonrpclib from txjsonrpc.web import jsonrpc -from txjsonrpc.web.jsonrpc import Handler +from txjsonrpc.web.jsonrpc import Handler, Proxy from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version @@ -47,6 +47,7 @@ from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES +from lbrynet.conf import SEARCH_SERVER from lbrynet.conf import DEFAULT_TIMEOUT, WALLET_TYPES from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob from lbrynet.core.Session import LBRYSession @@ -1228,6 +1229,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d def _get_est_cost(self, name): + log.info("Estimating cost for " + name) def _check_est(d, name): if isinstance(d.result, float): log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") @@ -1416,8 +1418,11 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) def _search(self, search): - d = self.session.wallet.get_nametrie() - d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) + proxy = Proxy(SEARCH_SERVER) + + d = proxy.callRemote('search', search) + # d = se.search(search) + # d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) return d def _render_response(self, result, code): @@ -1897,43 +1902,38 @@ class LBRYDaemon(jsonrpc.JSONRPC): t = [] for i in n: if i[0]: - if i[1][0][0] and i[1][1][0] and i[1][2][0]: - i[1][0][1]['value'] = str(i[1][0][1]['value']) - t.append([i[1][0][1], i[1][1][1], i[1][2][1]]) + tr = {} + tr.update(i[1][0]) + thumb = tr.get('thumbnail', None) + if thumb is None: + tr['thumbnail'] = "img/Free-speech-flag.svg" + tr['cost_est'] = i[1][1] + t.append(tr) return t - def resolve_claims(claims): - ds = [] - for claim in claims: - d1 = defer.succeed(claim) - d2 = self._resolve_name(claim['name']) - d3 = self._get_est_cost(claim['name']) - dl = defer.DeferredList([d1, d2, d3], consumeErrors=True) - ds.append(dl) - return defer.DeferredList(ds) + def get_est_costs(results): + def _get_costs(search_result): + log.info("**" + search_result['name']) + d = self._get_est_cost(search_result['name']) + d.addCallback(lambda p: _save_cost(search_result, p)) + return d - def _disp(results): - log.info('Found ' + str(len(results)) + ' search results') - consolidated_results = [] - for r in results: - t = {} - t.update(r[0]) - if not 'thumbnail' in r[1].keys(): - r[1]['thumbnail'] = "img/Free-speech-flag.svg" - t.update(r[1]) - t['cost_est'] = r[2] - consolidated_results.append(t) - # log.info(str(t)) + def _save_cost(value, cost): + log.info("Save cost") + log.info(value) + log.info(cost) + return [value, cost] - return consolidated_results + log.info("Estimating costs") + dl = defer.DeferredList([_get_costs(r) for r in results], consumeErrors=True) + return dl log.info('Search nametrie: ' + search) d = self._search(search) d.addCallback(lambda claims: claims[:self.max_search_results]) - d.addCallback(resolve_claims) + d.addCallback(get_est_costs) d.addCallback(_clean) - d.addCallback(_disp) d.addCallback(lambda results: self._render_response(results, OK_CODE)) return d From ba2e4df76ba0d95ce8edbde55bce67c3ebf696a6 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 21:31:19 -0400 Subject: [PATCH 65/73] remove debug stuff --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 09ff51351..9c64c1394 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1229,7 +1229,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d def _get_est_cost(self, name): - log.info("Estimating cost for " + name) def _check_est(d, name): if isinstance(d.result, float): log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") @@ -1419,11 +1418,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _search(self, search): proxy = Proxy(SEARCH_SERVER) - - d = proxy.callRemote('search', search) - # d = se.search(search) - # d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) - return d + return proxy.callRemote('search', search) def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) @@ -1919,12 +1914,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d def _save_cost(value, cost): - log.info("Save cost") - log.info(value) - log.info(cost) return [value, cost] - log.info("Estimating costs") dl = defer.DeferredList([_get_costs(r) for r in results], consumeErrors=True) return dl From bb3ba53c6ec181c3bb33ddb89ec40329e22c57f9 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 22:39:34 -0400 Subject: [PATCH 66/73] fix search results --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 9c64c1394..4efc11ec5 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1898,10 +1898,11 @@ class LBRYDaemon(jsonrpc.JSONRPC): for i in n: if i[0]: tr = {} - tr.update(i[1][0]) + tr.update(i[1][0]['value']) thumb = tr.get('thumbnail', None) if thumb is None: tr['thumbnail'] = "img/Free-speech-flag.svg" + tr['name'] = i[1][0]['name'] tr['cost_est'] = i[1][1] t.append(tr) return t From d21e458fe1abba2962a57fe5e76da5a8eda5cfae Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 29 Jul 2016 02:42:58 -0400 Subject: [PATCH 67/73] remove debug line --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 4efc11ec5..bc1cbba63 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1909,7 +1909,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): def get_est_costs(results): def _get_costs(search_result): - log.info("**" + search_result['name']) d = self._get_est_cost(search_result['name']) d.addCallback(lambda p: _save_cost(search_result, p)) return d From a574c33ac693e45e97439c33e5efe60f689e689a Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 29 Jul 2016 21:37:44 -0400 Subject: [PATCH 68/73] multiple search servers --- lbrynet/conf.py | 4 +++- lbrynet/lbrynet_daemon/LBRYDaemon.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 9576cc3b4..96e8b18dc 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -24,7 +24,9 @@ KNOWN_DHT_NODES = [('104.236.42.182', 4000), POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424' #POINTTRADER_SERVER = 'http://127.0.0.1:2424' -SEARCH_SERVER = "http://45.63.4.203:50005" +SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005", + "http://lighthouse2.lbry.io:50005", + "http://lighthouse3.lbry.io:50005"] LOG_FILE_NAME = "lbrynet.log" LOG_POST_URL = "https://lbry.io/log-upload" diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index bc1cbba63..f9b9e4425 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -47,7 +47,7 @@ from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES -from lbrynet.conf import SEARCH_SERVER +from lbrynet.conf import SEARCH_SERVERS from lbrynet.conf import DEFAULT_TIMEOUT, WALLET_TYPES from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob from lbrynet.core.Session import LBRYSession @@ -1417,7 +1417,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) def _search(self, search): - proxy = Proxy(SEARCH_SERVER) + proxy = Proxy(random.choice(SEARCH_SERVERS)) return proxy.callRemote('search', search) def _render_response(self, result, code): From 20a9e018f7bf574b965542b714f76ce414b29dd1 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 29 Jul 2016 22:36:30 -0400 Subject: [PATCH 69/73] revert updates to git --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 43 +++++++++++++++++----------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index f9b9e4425..deda28bec 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -148,8 +148,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.connected_to_internet = True self.connection_problem = None self.query_handlers = {} - self.pip_lbrynet_version = None - self.pip_lbryum_version = None + self.git_lbrynet_version = None + self.git_lbryum_version = None self.ui_version = None self.ip = None # TODO: this is confusing to set here, and then to be reset below. @@ -569,33 +569,42 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _check_remote_versions(self): def _get_lbryum_version(): try: - r = pkg_resources.get_distribution("lbryum").version - log.info("Local lbryum: %s" % lbryum_version) - log.info("Available lbryum: %s" % r) - self.pip_lbryum_version = r + r = urlopen("https://raw.githubusercontent.com/lbryio/lbryum/master/lib/version.py").read().split('\n') + version = next(line.split("=")[1].split("#")[0].replace(" ", "") + for line in r if "LBRYUM_VERSION" in line) + version = version.replace("'", "") + log.info( + "remote lbryum %s > local lbryum %s = %s", + version, lbryum_version, + utils.version_is_greater_than(version, lbryum_version) + ) + self.git_lbryum_version = version return defer.succeed(None) except: log.info("Failed to get lbryum version from git") - self.pip_lbryum_version = None + self.git_lbryum_version = None return defer.fail(None) def _get_lbrynet_version(): try: - r = pkg_resources.get_distribution("lbrynet").version - log.info("Local lbrynet: %s" % lbrynet_version) - log.info("Available lbrynet: %s" % r) - self.pip_lbrynet_version = r + version = get_lbrynet_version_from_github() + log.info( + "remote lbrynet %s > local lbrynet %s = %s", + version, lbrynet_version, + utils.version_is_greater_than(version, lbrynet_version) + ) + self.git_lbrynet_version = version return defer.succeed(None) except: log.info("Failed to get lbrynet version from git") - self.pip_lbrynet_version = None + self.git_lbrynet_version = None return defer.fail(None) d = _get_lbrynet_version() d.addCallback(lambda _: _get_lbryum_version()) def _check_connection_problems(self): - if not self.pip_lbrynet_version or not self.pip_lbryum_version: + if not self.git_lbrynet_version or not self.git_lbryum_version: self.connection_problem = CONNECTION_PROBLEM_CODES[0] elif self.startup_status[0] == 'loading_wallet': @@ -1537,10 +1546,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'lbrynet_version': lbrynet_version, 'lbryum_version': lbryum_version, 'ui_version': self.ui_version, - 'remote_lbrynet': self.pip_lbrynet_version, - 'remote_lbryum': self.pip_lbryum_version, - 'lbrynet_update_available': utils.version_is_greater_than(self.pip_lbrynet_version, lbrynet_version), - 'lbryum_update_available': utils.version_is_greater_than(self.pip_lbryum_version, lbryum_version), + 'remote_lbrynet': self.git_lbrynet_version, + 'remote_lbryum': self.git_lbryum_version, + 'lbrynet_update_available': utils.version_is_greater_than(self.git_lbrynet_version, lbrynet_version), + 'lbryum_update_available': utils.version_is_greater_than(self.git_lbryum_version, lbryum_version), } log.info("Get version info: " + json.dumps(msg)) From 64afb6d3050595958383ab8e21f7a0e40c3ba0cb Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 29 Jul 2016 22:43:50 -0400 Subject: [PATCH 70/73] clean up get_est_costs in search --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index deda28bec..e418477ba 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1917,15 +1917,12 @@ class LBRYDaemon(jsonrpc.JSONRPC): return t def get_est_costs(results): - def _get_costs(search_result): + def _save_cost(search_result): d = self._get_est_cost(search_result['name']) - d.addCallback(lambda p: _save_cost(search_result, p)) + d.addCallback(lambda p: [search_result, p]) return d - def _save_cost(value, cost): - return [value, cost] - - dl = defer.DeferredList([_get_costs(r) for r in results], consumeErrors=True) + dl = defer.DeferredList([_save_cost(r) for r in results], consumeErrors=True) return dl log.info('Search nametrie: ' + search) From 1e1d036590ad45e6de6e836935d07b2b7242dc33 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 29 Jul 2016 22:44:38 -0400 Subject: [PATCH 71/73] clean log line --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index e418477ba..70ebeae97 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1925,7 +1925,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): dl = defer.DeferredList([_save_cost(r) for r in results], consumeErrors=True) return dl - log.info('Search nametrie: ' + search) + log.info('Search: %s' % search) d = self._search(search) d.addCallback(lambda claims: claims[:self.max_search_results]) From cf8592ba12f0dd42ba13bf795b3d42782114e959 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 29 Jul 2016 22:49:50 -0400 Subject: [PATCH 72/73] docstring and todo --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 70ebeae97..cf0188a1f 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1889,7 +1889,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def jsonrpc_search_nametrie(self, p): """ - Search the nametrie for claims beginning with search (yes, this is a dumb search, it'll be made better) + Search the nametrie for claims Args: 'search': search query, string @@ -1897,6 +1897,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): List of search results """ + # TODO: change this function to "search", and use cached stream size info from the search server + if 'search' in p.keys(): search = p['search'] else: From 1ffcd67c9d2ba85b436d9ee5b5a0e1baa1a4ed29 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 29 Jul 2016 23:05:01 -0400 Subject: [PATCH 73/73] specify exception --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index cf0188a1f..ea9124d57 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1178,7 +1178,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _resolve_name(self, name, force_refresh=False): try: verify_name_characters(name) - except: + except AssertionError: log.error("Bad name") return defer.fail(InvalidNameError("Bad name"))