From d9e1e88c0552f31624e9f77b5442802c8330c368 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 29 Nov 2016 18:06:16 -0500 Subject: [PATCH 1/4] refactor get_est_cost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -add size parameter, if provided the sd blob won’t be downloaded (can be used with size information from lighthouse) -check if blob was already downloaded before trying to download it again --- lbrynet/conf.py | 2 +- lbrynet/lbrynet_daemon/Daemon.py | 89 ++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index c3c1910bb..733345dfb 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -141,8 +141,8 @@ ENVIRONMENT = Env( reflector_port=(int, 5566), download_timeout=(int, 30), max_search_results=(int, 25), - search_timeout=(float, 3.0), cache_time=(int, 150), + search_timeout=(float, 5.0), host_ui=(bool, True), check_ui_requirements=(bool, True), local_ui_path=(bool, False), diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index d687324f8..a0f3efc4f 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -17,6 +17,7 @@ from decimal import Decimal from twisted.web import server from twisted.internet import defer, threads, error, reactor, task from twisted.internet.task import LoopingCall +from twisted.python.failure import Failure from txjsonrpc import jsonrpclib from jsonschema import ValidationError @@ -819,7 +820,8 @@ class Daemon(AuthJSONRPCServer): self.session = Session(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid, blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port, known_dht_nodes=conf.settings.known_dht_nodes, peer_port=self.peer_port, - use_upnp=self.use_upnp, wallet=results['wallet']) + use_upnp=self.use_upnp, wallet=results['wallet'], + is_generous=conf.settings.is_generous_host) self.startup_status = STARTUP_STAGES[2] dl = defer.DeferredList([d1, d2], fireOnOneErrback=True) @@ -940,35 +942,60 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda _: log.info("Delete lbry file")) return d - def _get_est_cost(self, name): - def _check_est(d, name): - try: - if isinstance(d.result, float): - log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") - return defer.succeed(None) - except AttributeError: - pass - log.info("Timeout estimating cost for lbry://" + name + ", using key fee") - d.cancel() - return defer.succeed(None) + def _get_est_cost_from_name(self, name): + d = self._resolve_name(name) + d.addCallback(self._get_est_cost_from_metadata, name) + return d + + def _get_est_cost_from_metadata(self, metadata, name): + d = self._get_est_cost_from_sd_hash(metadata['sources']['lbry_sd_hash']) + + def _handle_err(err): + if isinstance(err, Failure): + log.warning("Timeout getting blob for cost est for lbry://%s, using only key fee", name) + return 0.0 + raise err + + d.addErrback(_handle_err) def _add_key_fee(data_cost): - d = self._resolve_name(name) - d.addCallback(lambda info: self.exchange_rate_manager.to_lbc(info.get('fee', None))) - d.addCallback(lambda fee: data_cost if fee is None else data_cost + fee.amount) - return d + fee = self.exchange_rate_manager.to_lbc(metadata.get('fee', None)) + return data_cost if fee is None else data_cost + fee.amount - d = self._resolve_name(name) - d.addCallback(lambda info: info['sources']['lbry_sd_hash']) - d.addCallback(lambda sd_hash: download_sd_blob(self.session, sd_hash, - self.blob_request_payment_rate_manager)) - d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) - d.addCallback(lambda metadata: metadata.validator.info_to_show()) - d.addCallback(lambda info: int(dict(info)['stream_size']) / 1000000 * self.data_rate) - d.addCallbacks(_add_key_fee, lambda _: _add_key_fee(0.0)) - reactor.callLater(self.search_timeout, _check_est, d, name) + d.addCallback(_add_key_fee) return d + def _get_sd_blob(self, blob, sd_hash): + if blob: + return self.session.blob_manager.get_blob(blob[0], True) + + def _check_est(downloader): + if downloader.result is not None: + downloader.cancel() + + d = defer.succeed(None) + reactor.callLater(self.search_timeout, _check_est, d) + d.addCallback(lambda _: download_sd_blob(self.session, sd_hash, self.blob_request_payment_rate_manager)) + return d + + def _get_size_from_sd_blob(self, sd_blob): + d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob) + d.addCallback(lambda metadata: metadata.validator.info_to_show()) + d.addCallback(lambda info: int(dict(info)['stream_size'])) + return d + + def _get_est_cost_from_sd_hash(self, sd_hash): + d = self.session.blob_manager.completed_blobs([sd_hash]) + d.addCallback(self._get_sd_blob, sd_hash) + d.addCallback(self._get_size_from_sd_blob) + d.addCallback(self._get_est_cost_from_stream_size) + return d + + def _get_est_cost_from_stream_size(self, size): + if self.session.payment_rate_manager.generous: + return 0.0 + return size / (10**6) * conf.settings.data_rate + def _get_lbry_file_by_uri(self, name): def _get_file(stream_info): sd = stream_info['sources']['lbry_sd_hash'] @@ -1563,17 +1590,23 @@ class Daemon(AuthJSONRPCServer): def jsonrpc_get_est_cost(self, p): """ - Get estimated cost for a lbry uri + Get estimated cost for a lbry stream Args: 'name': lbry uri + 'size': stream size, in bytes. if provided an sd blob won't be downloaded. Returns: estimated cost """ - name = p[FileID.NAME] + if 'size' in p: + size = p['size'] + d = defer.succeed(self._get_est_cost_from_stream_size(size)) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + return d - d = self._get_est_cost(name) + name = p[FileID.NAME] + d = self._get_est_cost_from_name(name) d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d From d93f86030cfb8f37e6d92da14718d63f0316ce14 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 30 Nov 2016 16:23:48 -0500 Subject: [PATCH 2/4] clean up get_est_cost functions --- lbrynet/lbrynet_daemon/Daemon.py | 107 ++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 38 deletions(-) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index a0f3efc4f..ba6c60bac 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -942,13 +942,59 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda _: log.info("Delete lbry file")) return d - def _get_est_cost_from_name(self, name): - d = self._resolve_name(name) - d.addCallback(self._get_est_cost_from_metadata, name) + def _get_or_download_sd_blob(self, blob, sd_hash): + if blob: + return self.session.blob_manager.get_blob(blob[0], True) + + def _check_est(downloader): + if downloader.result is not None: + downloader.cancel() + + d = defer.succeed(None) + reactor.callLater(self.search_timeout, _check_est, d) + d.addCallback(lambda _: download_sd_blob(self.session, sd_hash, self.blob_request_payment_rate_manager)) + return d + + def get_or_download_sd_blob(self, sd_hash): + """ + Return previously downloaded sd blob if already in the blob manager, otherwise download and return it + """ + + d = self.session.blob_manager.completed_blobs([sd_hash]) + d.addCallback(self._get_or_download_sd_blob, sd_hash) + return d + + def get_size_from_sd_blob(self, sd_blob): + """ + Get total stream size in bytes from a sd blob + """ + + d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob) + d.addCallback(lambda metadata: metadata.validator.info_to_show()) + d.addCallback(lambda info: int(dict(info)['stream_size'])) + return d + + def get_est_cost_from_stream_size(self, size): + """ + Calculate estimated LBC cost for a stream given its size in bytes + """ + + if self.session.payment_rate_manager.generous: + return 0.0 + return size / (10**6) * conf.settings.data_rate + + def get_est_cost_from_sd_hash(self, sd_hash): + """ + Get estimated cost from a sd hash + """ + + d = self.get_or_download_sd_blob(sd_hash) + d.addCallback(self.get_size_from_sd_blob) + d.addCallback(self.get_est_cost_from_stream_size) return d def _get_est_cost_from_metadata(self, metadata, name): - d = self._get_est_cost_from_sd_hash(metadata['sources']['lbry_sd_hash']) + d = self.get_est_cost_from_sd_hash(metadata['sources']['lbry_sd_hash']) def _handle_err(err): if isinstance(err, Failure): @@ -965,36 +1011,25 @@ class Daemon(AuthJSONRPCServer): d.addCallback(_add_key_fee) return d - def _get_sd_blob(self, blob, sd_hash): - if blob: - return self.session.blob_manager.get_blob(blob[0], True) + def get_est_cost_from_name(self, name): + """ + Resolve a name and return the estimated stream cost + """ - def _check_est(downloader): - if downloader.result is not None: - downloader.cancel() - - d = defer.succeed(None) - reactor.callLater(self.search_timeout, _check_est, d) - d.addCallback(lambda _: download_sd_blob(self.session, sd_hash, self.blob_request_payment_rate_manager)) + d = self._resolve_name(name) + d.addCallback(self._get_est_cost_from_metadata, name) return d - def _get_size_from_sd_blob(self, sd_blob): - d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob) - d.addCallback(lambda metadata: metadata.validator.info_to_show()) - d.addCallback(lambda info: int(dict(info)['stream_size'])) - return d - - def _get_est_cost_from_sd_hash(self, sd_hash): - d = self.session.blob_manager.completed_blobs([sd_hash]) - d.addCallback(self._get_sd_blob, sd_hash) - d.addCallback(self._get_size_from_sd_blob) - d.addCallback(self._get_est_cost_from_stream_size) - return d - - def _get_est_cost_from_stream_size(self, size): - if self.session.payment_rate_manager.generous: - return 0.0 - return size / (10**6) * conf.settings.data_rate + def get_est_cost(self, name=None, size=None): + """ + Get a cost estimate for a lbry stream, requires either a name to check or a given stream size in bytes + If no size is + """ + if name is None and size is None: + return defer.fail(Exception("Neither name nor size was provided")) + if size is not None: + return defer.succeed(self.get_est_cost_from_stream_size(size)) + return self.get_est_cost_from_name(name) def _get_lbry_file_by_uri(self, name): def _get_file(stream_info): @@ -1599,14 +1634,10 @@ class Daemon(AuthJSONRPCServer): estimated cost """ - if 'size' in p: - size = p['size'] - d = defer.succeed(self._get_est_cost_from_stream_size(size)) - d.addCallback(lambda r: self._render_response(r, OK_CODE)) - return d + size = p.get('size', None) + name = p.get(FileID.NAME, None) - name = p[FileID.NAME] - d = self._get_est_cost_from_name(name) + d = self.get_est_cost(name, size) d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d From b9bb337a07ad0d7d743cf27276b9a829ff1a0a32 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 2 Dec 2016 14:39:01 -0500 Subject: [PATCH 3/4] add tests for daemon.get_est_cost --- lbrynet/lbrynet_daemon/Daemon.py | 39 +++++++++----- tests/unit/lbrynet_daemon/test_Daemon.py | 68 +++++++++++++++++++++++- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index ba6c60bac..13535a767 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -974,7 +974,7 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda info: int(dict(info)['stream_size'])) return d - def get_est_cost_from_stream_size(self, size): + def _get_est_cost_from_stream_size(self, size): """ Calculate estimated LBC cost for a stream given its size in bytes """ @@ -983,6 +983,17 @@ class Daemon(AuthJSONRPCServer): return 0.0 return size / (10**6) * conf.settings.data_rate + def get_est_cost_using_known_size(self, name, size): + """ + Calculate estimated LBC cost for a stream given its size in bytes + """ + + cost = self._get_est_cost_from_stream_size(size) + + d = self._resolve_name(name) + d.addCallback(lambda metadata: self._add_key_fee_to_est_data_cost(metadata, cost)) + return d + def get_est_cost_from_sd_hash(self, sd_hash): """ Get estimated cost from a sd hash @@ -990,7 +1001,7 @@ class Daemon(AuthJSONRPCServer): d = self.get_or_download_sd_blob(sd_hash) d.addCallback(self.get_size_from_sd_blob) - d.addCallback(self.get_est_cost_from_stream_size) + d.addCallback(self._get_est_cost_from_stream_size) return d def _get_est_cost_from_metadata(self, metadata, name): @@ -1003,14 +1014,14 @@ class Daemon(AuthJSONRPCServer): raise err d.addErrback(_handle_err) - - def _add_key_fee(data_cost): - fee = self.exchange_rate_manager.to_lbc(metadata.get('fee', None)) - return data_cost if fee is None else data_cost + fee.amount - - d.addCallback(_add_key_fee) + d.addCallback(lambda data_cost: self._add_key_fee_to_est_data_cost(metadata, data_cost)) return d + def _add_key_fee_to_est_data_cost(self, metadata, data_cost): + fee = self.exchange_rate_manager.to_lbc(metadata.get('fee', None)) + fee_amount = 0.0 if fee is None else fee.amount + return data_cost + fee_amount + def get_est_cost_from_name(self, name): """ Resolve a name and return the estimated stream cost @@ -1020,15 +1031,15 @@ class Daemon(AuthJSONRPCServer): d.addCallback(self._get_est_cost_from_metadata, name) return d - def get_est_cost(self, name=None, size=None): + + def get_est_cost(self, name, size=None): """ - Get a cost estimate for a lbry stream, requires either a name to check or a given stream size in bytes - If no size is + Get a cost estimate for a lbry stream, if size is not provided the sd blob will be downloaded + to determine the stream size """ - if name is None and size is None: - return defer.fail(Exception("Neither name nor size was provided")) + if size is not None: - return defer.succeed(self.get_est_cost_from_stream_size(size)) + return self.get_est_cost_using_known_size(name, size) return self.get_est_cost_from_name(name) def _get_lbry_file_by_uri(self, name): diff --git a/tests/unit/lbrynet_daemon/test_Daemon.py b/tests/unit/lbrynet_daemon/test_Daemon.py index cb96e41a1..e6e34053f 100644 --- a/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/tests/unit/lbrynet_daemon/test_Daemon.py @@ -1,8 +1,14 @@ import mock import requests +from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker +from tests import util +from twisted.internet import defer from twisted.trial import unittest - from lbrynet.lbrynet_daemon import Daemon +from lbrynet.core import Session, PaymentRateManager +from lbrynet.lbrynet_daemon.Daemon import Daemon as LBRYDaemon +from lbrynet.lbrynet_daemon import ExchangeRateManager +from lbrynet import conf class MiscTests(unittest.TestCase): @@ -36,3 +42,63 @@ class MiscTests(unittest.TestCase): def test_error_is_thrown_when_version_cant_be_parsed(self): with self.assertRaises(Exception): Daemon.get_version_from_tag('garbage') + + +def get_test_daemon(data_rate=conf.settings.data_rate, generous=True, with_fee=False): + rates = { + 'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1}, + 'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2} + } + daemon = LBRYDaemon(None, None) + daemon.session = mock.Mock(spec=Session.Session) + daemon.exchange_rate_manager = ExchangeRateManager.DummyExchangeRateManager(rates) + base_prm = PaymentRateManager.BasePaymentRateManager(rate=data_rate) + prm = PaymentRateManager.NegotiatedPaymentRateManager(base_prm, DummyBlobAvailabilityTracker(), generous=generous) + daemon.session.payment_rate_manager = prm + metadata = { + "author": "extra", + "content_type": "video/mp4", + "description": "How did the ancient civilization of Sumer first develop the concept of the written word? It all began with simple warehouse tallies in the temples, but as the scribes sought more simple ways to record information, those tallies gradually evolved from pictograms into cuneiform text which could be used to convey complex, abstract, or even lyrical ideas.", + "language": "en", + "license": "Creative Commons Attribution 3.0 United States", + "license_url": "https://creativecommons.org/licenses/by/3.0/us/legalcode", + "nsfw": False, + "sources": { + "lbry_sd_hash": "d2b8b6e907dde95245fe6d144d16c2fdd60c4e0c6463ec98b85642d06d8e9414e8fcfdcb7cb13532ec5454fb8fe7f280"}, + "thumbnail": "http://i.imgur.com/HFSRkKw.png", + "title": "The History of Writing - Where the Story Begins", + "ver": "0.0.3" + } + if with_fee: + metadata.update({"fee": {"USD": {"address": "bQ6BGboPV2SpTMEP7wLNiAcnsZiH8ye6eA", "amount": 0.75}}}) + daemon._resolve_name = lambda x: defer.succeed(metadata) + return daemon + + +class TestCostEst(unittest.TestCase): + def setUp(self): + util.resetTime(self) + + def test_cost_est_with_fee_and_generous(self): + size = 10000000 + fake_fee_amount = 4.5 + daemon = get_test_daemon(generous=True, with_fee=True) + self.assertEquals(daemon.get_est_cost("test", size).result, fake_fee_amount) + + def test_cost_est_with_fee_and_not_generous(self): + size = 10000000 + fake_fee_amount = 4.5 + data_rate = conf.settings.data_rate + daemon = get_test_daemon(generous=False, with_fee=True) + self.assertEquals(daemon.get_est_cost("test", size).result, (size / (10**6) * data_rate) + fake_fee_amount) + + def test_data_cost_with_generous(self): + size = 10000000 + daemon = get_test_daemon(generous=True) + self.assertEquals(daemon.get_est_cost("test", size).result, 0.0) + + def test_data_cost_with_non_generous(self): + size = 10000000 + data_rate = conf.settings.data_rate + daemon = get_test_daemon(generous=False) + self.assertEquals(daemon.get_est_cost("test", size).result, (size / (10**6) * data_rate)) From 948be0f0409e75dbbb38ef592661064522925aea Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 4 Dec 2016 16:18:13 -0500 Subject: [PATCH 4/4] clean up tests --- tests/unit/lbrynet_daemon/test_Daemon.py | 46 +++++++++++++----------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tests/unit/lbrynet_daemon/test_Daemon.py b/tests/unit/lbrynet_daemon/test_Daemon.py index e6e34053f..495162f5e 100644 --- a/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/tests/unit/lbrynet_daemon/test_Daemon.py @@ -44,7 +44,10 @@ class MiscTests(unittest.TestCase): Daemon.get_version_from_tag('garbage') -def get_test_daemon(data_rate=conf.settings.data_rate, generous=True, with_fee=False): +def get_test_daemon(data_rate=None, generous=True, with_fee=False): + if data_rate is None: + data_rate = conf.settings.data_rate + rates = { 'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1}, 'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2} @@ -56,22 +59,22 @@ def get_test_daemon(data_rate=conf.settings.data_rate, generous=True, with_fee=F prm = PaymentRateManager.NegotiatedPaymentRateManager(base_prm, DummyBlobAvailabilityTracker(), generous=generous) daemon.session.payment_rate_manager = prm metadata = { - "author": "extra", - "content_type": "video/mp4", - "description": "How did the ancient civilization of Sumer first develop the concept of the written word? It all began with simple warehouse tallies in the temples, but as the scribes sought more simple ways to record information, those tallies gradually evolved from pictograms into cuneiform text which could be used to convey complex, abstract, or even lyrical ideas.", - "language": "en", - "license": "Creative Commons Attribution 3.0 United States", - "license_url": "https://creativecommons.org/licenses/by/3.0/us/legalcode", + "author": "fake author", + "content_type": "fake/format", + "description": "fake description", + "license": "fake license", + "license_url": "fake license url", "nsfw": False, "sources": { - "lbry_sd_hash": "d2b8b6e907dde95245fe6d144d16c2fdd60c4e0c6463ec98b85642d06d8e9414e8fcfdcb7cb13532ec5454fb8fe7f280"}, - "thumbnail": "http://i.imgur.com/HFSRkKw.png", - "title": "The History of Writing - Where the Story Begins", + "lbry_sd_hash": "d2b8b6e907dde95245fe6d144d16c2fdd60c4e0c6463ec98b85642d06d8e9414e8fcfdcb7cb13532ec5454fb8fe7f280" + }, + "thumbnail": "fake thumbnail", + "title": "fake title", "ver": "0.0.3" } if with_fee: metadata.update({"fee": {"USD": {"address": "bQ6BGboPV2SpTMEP7wLNiAcnsZiH8ye6eA", "amount": 0.75}}}) - daemon._resolve_name = lambda x: defer.succeed(metadata) + daemon._resolve_name = lambda _: defer.succeed(metadata) return daemon @@ -79,26 +82,29 @@ class TestCostEst(unittest.TestCase): def setUp(self): util.resetTime(self) - def test_cost_est_with_fee_and_generous(self): + def test_fee_and_generous_data(self): size = 10000000 - fake_fee_amount = 4.5 + correct_result = 4.5 daemon = get_test_daemon(generous=True, with_fee=True) - self.assertEquals(daemon.get_est_cost("test", size).result, fake_fee_amount) + self.assertEquals(daemon.get_est_cost("test", size).result, correct_result) - def test_cost_est_with_fee_and_not_generous(self): + def test_fee_and_ungenerous_data(self): size = 10000000 fake_fee_amount = 4.5 data_rate = conf.settings.data_rate + correct_result = size / 10**6 * data_rate + fake_fee_amount daemon = get_test_daemon(generous=False, with_fee=True) - self.assertEquals(daemon.get_est_cost("test", size).result, (size / (10**6) * data_rate) + fake_fee_amount) + self.assertEquals(daemon.get_est_cost("test", size).result, correct_result) - def test_data_cost_with_generous(self): + def test_generous_data_and_no_fee(self): size = 10000000 + correct_result = 0.0 daemon = get_test_daemon(generous=True) - self.assertEquals(daemon.get_est_cost("test", size).result, 0.0) + self.assertEquals(daemon.get_est_cost("test", size).result, correct_result) - def test_data_cost_with_non_generous(self): + def test_ungenerous_data_and_no_fee(self): size = 10000000 data_rate = conf.settings.data_rate + correct_result = size / 10**6 * data_rate daemon = get_test_daemon(generous=False) - self.assertEquals(daemon.get_est_cost("test", size).result, (size / (10**6) * data_rate)) + self.assertEquals(daemon.get_est_cost("test", size).result, correct_result)