LBRYFee things

-move exchange rate updates to wallet, do them every half hour
-add convention checker for fees
-pay BTC or USD denominated key fees in converted LBC amount

lbrynet/core/LBRYMetadata.py

@@ -1,63 +1,105 @@
 import requests
 import json
+import time
 
 from googlefinance import getQuotes
 from lbrynet.conf import CURRENCIES
+import logging
+
+log = logging.getLogger(__name__)
+
+BITTREX_FEE = 0.0025
 
 SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
-
 BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources']
 OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey']
 
 #v0.0.1 metadata
 METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}}
-
 #v0.0.2 metadata additions
 METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['licence_url']}
-
 CURRENT_METADATA_VERSION = '0.0.2'
 
+
+#v0.0.1 fee
+FEE_REVISIONS = {'0.0.1': {'required': ['amount', 'address'], 'optional': []}}
+CURRENT_FEE_REVISION = '0.0.1'
+
+
 class LBRYFee(object):
-    def __init__(self, currency, amount, address=None):
+    def __init__(self, fee_dict, rate_dict):
-        assert currency in [c.keys()[0] for c in CURRENCIES], "Unsupported currency: %s" % str(currency)
+        fee = LBRYFeeFormat(fee_dict)
-        self.address = address
-        self.currency_symbol = currency
-        self.currency = next(c for c in CURRENCIES if self.currency_symbol in c)
-        if not isinstance(amount, float):
-            self.amount = float(amount)
-        else:
-            self.amount = amount
 
-    def __call__(self):
+        for currency in fee:
-        return {self.currency_symbol: {'amount': self.amount, 'address': self.address}}
+            self.address = fee[currency]['address']
+            if not isinstance(fee[currency]['amount'], float):
+                self.amount = float(fee[currency]['amount'])
+            else:
+                self.amount = fee[currency]['amount']
+            self.currency_symbol = currency
 
-    def convert(self, amount_only=False):
+        assert 'BTCLBC' in rate_dict and 'USDBTC' in rate_dict
+        for fx in rate_dict:
+            assert int(time.time()) - int(rate_dict[fx]['ts']) < 3600, "%s quote is out of date" % fx
+        self._USDBTC = {'spot': rate_dict['USDBTC']['spot'], 'ts': rate_dict['USDBTC']['ts']}
+        self._BTCLBC = {'spot': rate_dict['BTCLBC']['spot'], 'ts': rate_dict['BTCLBC']['ts']}
+
+    def to_lbc(self):
+        r = None
         if self.currency_symbol == "LBC":
             r = round(float(self.amount), 5)
         elif self.currency_symbol == "BTC":
-            r = round(float(self.BTC_to_LBC(self.amount)), 5)
+            r = round(float(self._btc_to_lbc(self.amount)), 5)
         elif self.currency_symbol == "USD":
-            r = round(float(self.BTC_to_LBC(self.USD_to_BTC(self.amount))), 5)
+            r = round(float(self._btc_to_lbc(self._usd_to_btc(self.amount))), 5)
-
+        assert r is not None
-        if not amount_only:
-            return {'LBC': {'amount': r, 'address': self.address}}
-        else:
-            return r
-
-    def USD_to_BTC(self, usd):
-        r = float(getQuotes('CURRENCY:%sBTC' % self.currency_symbol)[0]['LastTradePrice']) * float(usd)
         return r
 
-    def BTC_to_LBC(self, btc):
+    def to_usd(self):
-        r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'})
+        r = None
-        last = json.loads(r.text)['result']['Last']
+        if self.currency_symbol == "USD":
-        converted = float(btc) / float(last)
+            r = round(float(self.amount), 5)
-        return converted
+        elif self.currency_symbol == "BTC":
+            r = round(float(self._btc_to_usd(self.amount)), 5)
+        elif self.currency_symbol == "LBC":
+            r = round(float(self._btc_to_usd(self._lbc_to_btc(self.amount))), 5)
+        assert r is not None
+        return r
+
+    def _usd_to_btc(self, usd):
+        return self._USDBTC['spot'] * float(usd)
+
+    def _btc_to_usd(self, btc):
+        return float(btc) / self._USDBTC['spot']
+
+    def _btc_to_lbc(self, btc):
+        return float(btc) / self._BTCLBC['spot'] / (1.0 - BITTREX_FEE)
+
+    def _lbc_to_btc(self, lbc):
+        return self._BTCLBC['spot'] * float(lbc)
 
 
-def fee_from_dict(fee_dict):
+class LBRYFeeFormat(dict):
-    s = fee_dict.keys()[0]
+    def __init__(self, fee_dict):
-    return LBRYFee(s, fee_dict[s]['amount'], fee_dict[s]['address'])
+        dict.__init__(self)
+        self.fee_version = None
+        f = fee_dict.copy()
+        assert len(fee_dict) == 1
+        for currency in fee_dict:
+            assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency)
+            self.update({currency: {}})
+
+            for version in FEE_REVISIONS:
+                for k in FEE_REVISIONS[version]['required']:
+                    assert k in fee_dict, "Missing required fee field: %s" % k
+                    self[currency].update({k: f.pop(k)})
+                for k in FEE_REVISIONS[version]['optional']:
+                    if k in fee_dict:
+                        self[currency].update({k: f.pop(k)})
+                if not len(f):
+                    self.fee_version = version
+                    break
+            assert f == {}, "Unknown fee keys: %s" % json.dumps(f.keys())
 
 
 class Metadata(dict):
@@ -66,9 +108,6 @@ class Metadata(dict):
         self.metaversion = None
         m = metadata.copy()
 
-        if 'fee' in metadata:
-            assert fee_from_dict(metadata['fee'])
-
         assert "sources" in metadata, "No sources given"
         for source in metadata['sources']:
             assert source in SOURCE_TYPES, "Unknown source type"
@@ -78,9 +117,13 @@ class Metadata(dict):
                 assert k in metadata, "Missing required metadata field: %s" % k
                 self.update({k: m.pop(k)})
             for k in METADATA_REVISIONS[version]['optional']:
-                if k in metadata:
+                if k == 'fee':
+                    pass
+                elif k in metadata:
                     self.update({k: m.pop(k)})
-            if not len(m):
+            if not len(m) or m.keys() == ['fee']:
                 self.metaversion = version
                 break
+        if 'fee' in m:
+            self.update({'fee': LBRYFeeFormat(m.pop('fee'))})
         assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys())

lbrynet/core/LBRYWallet.py

@@ -1,17 +1,12 @@
 import sys
-from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
+import datetime
-from lbrynet.core.client.ClientRequest import ClientRequest
+import logging
-from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
+import json
-from lbrynet.core.Error import InsufficientFundsError
+import subprocess
-from lbrynet.core.sqlite_helpers import rerun_if_locked
+import socket
-from lbrynet.conf import SOURCE_TYPES
+import time
-from lbrynet.core.LBRYMetadata import Metadata
+import os
-
+import requests
-from lbryum import SimpleConfig, Network
-from lbryum.lbrycrd import COIN, TYPE_ADDRESS
-from lbryum.wallet import WalletStorage, Wallet
-from lbryum.commands import known_commands, Commands
-from lbryum.transaction import Transaction
 
 from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
 from twisted.internet import threads, reactor, defer, task
@@ -20,15 +15,24 @@ from twisted.enterprise import adbapi
 from collections import defaultdict, deque
 from zope.interface import implements
 from decimal import Decimal
-import datetime
+from googlefinance import getQuotes
-import logging
+
-import json
+from lbryum import SimpleConfig, Network
-import subprocess
+from lbryum.lbrycrd import COIN, TYPE_ADDRESS
-import socket
+from lbryum.wallet import WalletStorage, Wallet
-import time
+from lbryum.commands import known_commands, Commands
-import os
+from lbryum.transaction import Transaction
+
+from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
+from lbrynet.core.client.ClientRequest import ClientRequest
+from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
+from lbrynet.core.Error import InsufficientFundsError
+from lbrynet.core.sqlite_helpers import rerun_if_locked
+from lbrynet.conf import SOURCE_TYPES
+from lbrynet.core.LBRYMetadata import Metadata
 
 log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
 alert = logging.getLogger("lbryalert." + __name__)
 
 
@@ -80,6 +84,50 @@ class LBRYWallet(object):
         self._batch_count = 20
         self._first_run = self._FIRST_RUN_UNKNOWN
 
+        self._USDBTC = None
+        self._BTCLBC = None
+        self._exchange_rate_updater = task.LoopingCall(self._update_exchange_rates)
+
+    def _usd_to_btc(self):
+        if self._USDBTC is not None:
+            if int(time.time()) - int(self._USDBTC['ts']) < 600:
+                log.info("USDBTC quote is new enough")
+                return defer.succeed({})
+
+        log.info("Getting new USDBTC quote")
+        x = float(getQuotes('CURRENCY:USDBTC')[0]['LastTradePrice'])
+        return defer.succeed({'USDBTC': {'spot': x, 'ts': int(time.time())}})
+
+    def _btc_to_lbc(self):
+        if self._BTCLBC is not None:
+            if int(time.time()) - int(self._BTCLBC['ts']) < 600:
+                log.info("BTCLBC quote is new enough")
+                return defer.succeed({})
+
+        log.info("Getting new BTCLBC quote")
+        r = requests.get("https://bittrex.com/api/v1.1/public/getmarkethistory", {'market': 'BTC-LBC', 'count': 50})
+        trades = json.loads(r.text)['result']
+        vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades])
+        x = (1.0 / float(vwap)) / 0.99975
+
+        return defer.succeed({'BTCLBC': {'spot': x, 'ts': int(time.time())}})
+
+    def _set_exchange_rates(self, rates):
+        if 'USDBTC' in rates:
+            assert int(time.time()) - int(rates['USDBTC']['ts']) < 3600, "new USDBTC quote is too old"
+            self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']}
+            log.info("Updated USDBTC rate: %s" % json.dumps(self._USDBTC))
+        if 'BTCLBC' in rates:
+            assert int(time.time()) - int(rates['BTCLBC']['ts']) < 3600, "new BTCLBC quote is too old"
+            self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']}
+            log.info("Updated BTCLBC rate: %s" % json.dumps(self._BTCLBC))
+
+    def _update_exchange_rates(self):
+        d = self._usd_to_btc()
+        d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates))
+        d.addCallback(lambda _: self._btc_to_lbc())
+        d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates))
+
     def start(self):
 
         def start_manage():
@@ -87,6 +135,8 @@ class LBRYWallet(object):
             self.manage()
             return True
 
+        self._exchange_rate_updater.start(1800)
+
         d = self._open_db()
         d.addCallback(lambda _: self._start())
         d.addCallback(lambda _: start_manage())

lbrynet/lbrynet_daemon/LBRYDownloader.py

@@ -11,7 +11,7 @@ from twisted.internet.task import LoopingCall
 from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError
 from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.core.StreamDescriptor import download_sd_blob
-from lbrynet.core.LBRYMetadata import Metadata, fee_from_dict
+from lbrynet.core.LBRYMetadata import Metadata, LBRYFee
 from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
 from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME
 
@@ -90,9 +90,9 @@ class GetStream(object):
         self.stream_hash = self.metadata['sources']['lbry_sd_hash']
 
         if 'fee' in self.metadata:
-            fee = fee_from_dict(self.metadata['fee'])
+            self.fee = LBRYFee(self.metadata['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC})
-            if fee.convert(amount_only=True) > self.max_key_fee:
+            if self.fee.to_lbc() > self.max_key_fee:
-                log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name))
+                log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name))
                 return defer.fail(None)
 
         def _cause_timeout():
@@ -124,11 +124,12 @@ class GetStream(object):
     def _start_download(self, downloader):
         def _pay_key_fee():
             if self.fee is not None:
-                reserved_points = self.wallet.reserve_points(self.fee.address, self.fee.convert(amount_only=True))
+                x = self.fee.to_lbc()
+                reserved_points = self.wallet.reserve_points(self.fee.address, x)
                 if reserved_points is None:
                     return defer.fail(InsufficientFundsError())
-                log.info("Key fee: %f --> %s" % (self.key_fee, self.key_fee_address))
+                log.info("Key fee: %f --> %s" % (x, self.fee.address))
-                return self.wallet.send_points_to_address(reserved_points, self.key_fee)
+                return self.wallet.send_points_to_address(reserved_points, self.fee.address)
             return defer.succeed(None)
 
         d = _pay_key_fee()