Merge pull request #294 from lbryio/approximate-stream-price

Add 'size' parameter to get_est_cost

lbrynet/conf.py

@@ -141,8 +141,8 @@ ENVIRONMENT = Env(
     reflector_port=(int, 5566),
     download_timeout=(int, 30),
     max_search_results=(int, 25),
-    search_timeout=(float, 3.0),
     cache_time=(int, 150),
+    search_timeout=(float, 5.0),
     host_ui=(bool, True),
     check_ui_requirements=(bool, True),
     local_ui_path=(bool, False),

lbrynet/lbrynet_daemon/Daemon.py

@@ -17,6 +17,7 @@ from decimal import Decimal
 from twisted.web import server
 from twisted.internet import defer, threads, error, reactor, task
 from twisted.internet.task import LoopingCall
+from twisted.python.failure import Failure
 from txjsonrpc import jsonrpclib
 from jsonschema import ValidationError
 
@@ -819,7 +820,8 @@ class Daemon(AuthJSONRPCServer):
             self.session = Session(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid,
                                        blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port,
                                        known_dht_nodes=conf.settings.known_dht_nodes, peer_port=self.peer_port,
-                                       use_upnp=self.use_upnp, wallet=results['wallet'])
+                                       use_upnp=self.use_upnp, wallet=results['wallet'],
+                                       is_generous=conf.settings.is_generous_host)
             self.startup_status = STARTUP_STAGES[2]
 
         dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
@@ -940,35 +942,106 @@ class Daemon(AuthJSONRPCServer):
         d.addCallback(lambda _: log.info("Delete lbry file"))
         return d
 
-    def _get_est_cost(self, name):
+    def _get_or_download_sd_blob(self, blob, sd_hash):
-        def _check_est(d, name):
+        if blob:
-            try:
+            return self.session.blob_manager.get_blob(blob[0], True)
-                if isinstance(d.result, float):
-                    log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC")
-                    return defer.succeed(None)
-            except AttributeError:
-                pass
-            log.info("Timeout estimating cost for lbry://" + name + ", using key fee")
-            d.cancel()
-            return defer.succeed(None)
 
-        def _add_key_fee(data_cost):
+        def _check_est(downloader):
-            d = self._resolve_name(name)
+            if downloader.result is not None:
-            d.addCallback(lambda info: self.exchange_rate_manager.to_lbc(info.get('fee', None)))
+                downloader.cancel()
-            d.addCallback(lambda fee: data_cost if fee is None else data_cost + fee.amount)
+
-            return d
+        d = defer.succeed(None)
+        reactor.callLater(self.search_timeout, _check_est, d)
+        d.addCallback(lambda _: download_sd_blob(self.session, sd_hash, self.blob_request_payment_rate_manager))
+        return d
+
+    def get_or_download_sd_blob(self, sd_hash):
+        """
+        Return previously downloaded sd blob if already in the blob manager, otherwise download and return it
+        """
+
+        d = self.session.blob_manager.completed_blobs([sd_hash])
+        d.addCallback(self._get_or_download_sd_blob, sd_hash)
+        return d
+
+    def get_size_from_sd_blob(self, sd_blob):
+        """
+        Get total stream size in bytes from a sd blob
+        """
+
+        d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
+        d.addCallback(lambda metadata: metadata.validator.info_to_show())
+        d.addCallback(lambda info: int(dict(info)['stream_size']))
+        return d
+
+    def _get_est_cost_from_stream_size(self, size):
+        """
+        Calculate estimated LBC cost for a stream given its size in bytes
+        """
+
+        if self.session.payment_rate_manager.generous:
+            return 0.0
+        return size / (10**6) * conf.settings.data_rate
+
+    def get_est_cost_using_known_size(self, name, size):
+        """
+        Calculate estimated LBC cost for a stream given its size in bytes
+        """
+
+        cost = self._get_est_cost_from_stream_size(size)
 
         d = self._resolve_name(name)
-        d.addCallback(lambda info: info['sources']['lbry_sd_hash'])
+        d.addCallback(lambda metadata: self._add_key_fee_to_est_data_cost(metadata, cost))
-        d.addCallback(lambda sd_hash: download_sd_blob(self.session, sd_hash,
-                                                    self.blob_request_payment_rate_manager))
-        d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
-        d.addCallback(lambda metadata: metadata.validator.info_to_show())
-        d.addCallback(lambda info: int(dict(info)['stream_size']) / 1000000 * self.data_rate)
-        d.addCallbacks(_add_key_fee, lambda _: _add_key_fee(0.0))
-        reactor.callLater(self.search_timeout, _check_est, d, name)
         return d
 
+    def get_est_cost_from_sd_hash(self, sd_hash):
+        """
+        Get estimated cost from a sd hash
+        """
+
+        d = self.get_or_download_sd_blob(sd_hash)
+        d.addCallback(self.get_size_from_sd_blob)
+        d.addCallback(self._get_est_cost_from_stream_size)
+        return d
+
+    def _get_est_cost_from_metadata(self, metadata, name):
+        d = self.get_est_cost_from_sd_hash(metadata['sources']['lbry_sd_hash'])
+
+        def _handle_err(err):
+            if isinstance(err, Failure):
+                log.warning("Timeout getting blob for cost est for lbry://%s, using only key fee", name)
+                return 0.0
+            raise err
+
+        d.addErrback(_handle_err)
+        d.addCallback(lambda data_cost: self._add_key_fee_to_est_data_cost(metadata, data_cost))
+        return d
+
+    def _add_key_fee_to_est_data_cost(self, metadata, data_cost):
+        fee = self.exchange_rate_manager.to_lbc(metadata.get('fee', None))
+        fee_amount = 0.0 if fee is None else fee.amount
+        return data_cost + fee_amount
+
+    def get_est_cost_from_name(self, name):
+        """
+        Resolve a name and return the estimated stream cost
+        """
+
+        d = self._resolve_name(name)
+        d.addCallback(self._get_est_cost_from_metadata, name)
+        return d
+
+
+    def get_est_cost(self, name, size=None):
+        """
+        Get a cost estimate for a lbry stream, if size is not provided the sd blob will be downloaded
+        to determine the stream size
+        """
+
+        if size is not None:
+            return self.get_est_cost_using_known_size(name, size)
+        return self.get_est_cost_from_name(name)
+
     def _get_lbry_file_by_uri(self, name):
         def _get_file(stream_info):
             sd = stream_info['sources']['lbry_sd_hash']
@@ -1563,17 +1636,19 @@ class Daemon(AuthJSONRPCServer):
 
     def jsonrpc_get_est_cost(self, p):
         """
-        Get estimated cost for a lbry uri
+        Get estimated cost for a lbry stream
 
         Args:
             'name': lbry uri
+            'size': stream size, in bytes. if provided an sd blob won't be downloaded.
         Returns:
             estimated cost
         """
 
-        name = p[FileID.NAME]
+        size = p.get('size', None)
+        name = p.get(FileID.NAME, None)
 
-        d = self._get_est_cost(name)
+        d = self.get_est_cost(name, size)
         d.addCallback(lambda r: self._render_response(r, OK_CODE))
         return d
 

tests/unit/lbrynet_daemon/test_Daemon.py

@@ -1,8 +1,14 @@
 import mock
 import requests
+from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker
+from tests import util
+from twisted.internet import defer
 from twisted.trial import unittest
-
 from lbrynet.lbrynet_daemon import Daemon
+from lbrynet.core import Session, PaymentRateManager
+from lbrynet.lbrynet_daemon.Daemon import Daemon as LBRYDaemon
+from lbrynet.lbrynet_daemon import ExchangeRateManager
+from lbrynet import conf
 
 
 class MiscTests(unittest.TestCase):
@@ -36,3 +42,69 @@ class MiscTests(unittest.TestCase):
     def test_error_is_thrown_when_version_cant_be_parsed(self):
         with self.assertRaises(Exception):
             Daemon.get_version_from_tag('garbage')
+
+
+def get_test_daemon(data_rate=None, generous=True, with_fee=False):
+    if data_rate is None:
+        data_rate = conf.settings.data_rate
+
+    rates = {
+                'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1},
+                'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2}
+            }
+    daemon = LBRYDaemon(None, None)
+    daemon.session = mock.Mock(spec=Session.Session)
+    daemon.exchange_rate_manager = ExchangeRateManager.DummyExchangeRateManager(rates)
+    base_prm = PaymentRateManager.BasePaymentRateManager(rate=data_rate)
+    prm = PaymentRateManager.NegotiatedPaymentRateManager(base_prm, DummyBlobAvailabilityTracker(), generous=generous)
+    daemon.session.payment_rate_manager = prm
+    metadata = {
+        "author": "fake author",
+        "content_type": "fake/format",
+        "description": "fake description",
+        "license": "fake license",
+        "license_url": "fake license url",
+        "nsfw": False,
+        "sources": {
+            "lbry_sd_hash": "d2b8b6e907dde95245fe6d144d16c2fdd60c4e0c6463ec98b85642d06d8e9414e8fcfdcb7cb13532ec5454fb8fe7f280"
+        },
+        "thumbnail": "fake thumbnail",
+        "title": "fake title",
+        "ver": "0.0.3"
+    }
+    if with_fee:
+        metadata.update({"fee": {"USD": {"address": "bQ6BGboPV2SpTMEP7wLNiAcnsZiH8ye6eA", "amount": 0.75}}})
+    daemon._resolve_name = lambda _: defer.succeed(metadata)
+    return daemon
+
+
+class TestCostEst(unittest.TestCase):
+    def setUp(self):
+        util.resetTime(self)
+
+    def test_fee_and_generous_data(self):
+        size = 10000000
+        correct_result = 4.5
+        daemon = get_test_daemon(generous=True, with_fee=True)
+        self.assertEquals(daemon.get_est_cost("test", size).result, correct_result)
+
+    def test_fee_and_ungenerous_data(self):
+        size = 10000000
+        fake_fee_amount = 4.5
+        data_rate = conf.settings.data_rate
+        correct_result = size / 10**6 * data_rate + fake_fee_amount
+        daemon = get_test_daemon(generous=False, with_fee=True)
+        self.assertEquals(daemon.get_est_cost("test", size).result, correct_result)
+
+    def test_generous_data_and_no_fee(self):
+        size = 10000000
+        correct_result = 0.0
+        daemon = get_test_daemon(generous=True)
+        self.assertEquals(daemon.get_est_cost("test", size).result, correct_result)
+
+    def test_ungenerous_data_and_no_fee(self):
+        size = 10000000
+        data_rate = conf.settings.data_rate
+        correct_result = size / 10**6 * data_rate
+        daemon = get_test_daemon(generous=False)
+        self.assertEquals(daemon.get_est_cost("test", size).result, correct_result)