From c4a78a149bd3b3f1f021e5d32c6766ce89424a41 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 25 Jul 2016 21:45:42 -0400 Subject: [PATCH] LBRYFee things -move exchange rate updates to wallet, do them every half hour -add convention checker for fees -pay BTC or USD denominated key fees in converted LBC amount --- lbrynet/core/LBRYMetadata.py | 119 +++++++++++++++-------- lbrynet/core/LBRYWallet.py | 90 +++++++++++++---- lbrynet/lbrynet_daemon/LBRYDownloader.py | 15 +-- 3 files changed, 159 insertions(+), 65 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index efbb334de..4090a2f2a 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -1,63 +1,105 @@ import requests import json +import time from googlefinance import getQuotes from lbrynet.conf import CURRENCIES +import logging + +log = logging.getLogger(__name__) + +BITTREX_FEE = 0.0025 SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] - BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] #v0.0.1 metadata METADATA_REVISIONS = {'0.0.1': {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}} - #v0.0.2 metadata additions METADATA_REVISIONS['0.0.2'] = {'required': ['nsfw', 'ver'], 'optional': ['licence_url']} - CURRENT_METADATA_VERSION = '0.0.2' + +#v0.0.1 fee +FEE_REVISIONS = {'0.0.1': {'required': ['amount', 'address'], 'optional': []}} +CURRENT_FEE_REVISION = '0.0.1' + + class LBRYFee(object): - def __init__(self, currency, amount, address=None): - assert currency in [c.keys()[0] for c in CURRENCIES], "Unsupported currency: %s" % str(currency) - self.address = address - self.currency_symbol = currency - self.currency = next(c for c in CURRENCIES if self.currency_symbol in c) - if not isinstance(amount, float): - self.amount = float(amount) - else: - self.amount = amount + def __init__(self, fee_dict, rate_dict): + fee = LBRYFeeFormat(fee_dict) - def __call__(self): - return {self.currency_symbol: {'amount': self.amount, 'address': self.address}} + for currency in fee: + self.address = fee[currency]['address'] + if not isinstance(fee[currency]['amount'], float): + self.amount = float(fee[currency]['amount']) + else: + self.amount = fee[currency]['amount'] + self.currency_symbol = currency - def convert(self, amount_only=False): + assert 'BTCLBC' in rate_dict and 'USDBTC' in rate_dict + for fx in rate_dict: + assert int(time.time()) - int(rate_dict[fx]['ts']) < 3600, "%s quote is out of date" % fx + self._USDBTC = {'spot': rate_dict['USDBTC']['spot'], 'ts': rate_dict['USDBTC']['ts']} + self._BTCLBC = {'spot': rate_dict['BTCLBC']['spot'], 'ts': rate_dict['BTCLBC']['ts']} + + def to_lbc(self): + r = None if self.currency_symbol == "LBC": r = round(float(self.amount), 5) elif self.currency_symbol == "BTC": - r = round(float(self.BTC_to_LBC(self.amount)), 5) + r = round(float(self._btc_to_lbc(self.amount)), 5) elif self.currency_symbol == "USD": - r = round(float(self.BTC_to_LBC(self.USD_to_BTC(self.amount))), 5) - - if not amount_only: - return {'LBC': {'amount': r, 'address': self.address}} - else: - return r - - def USD_to_BTC(self, usd): - r = float(getQuotes('CURRENCY:%sBTC' % self.currency_symbol)[0]['LastTradePrice']) * float(usd) + r = round(float(self._btc_to_lbc(self._usd_to_btc(self.amount))), 5) + assert r is not None return r - def BTC_to_LBC(self, btc): - r = requests.get("https://bittrex.com/api/v1.1/public/getticker", {'market': 'BTC-LBC'}) - last = json.loads(r.text)['result']['Last'] - converted = float(btc) / float(last) - return converted + def to_usd(self): + r = None + if self.currency_symbol == "USD": + r = round(float(self.amount), 5) + elif self.currency_symbol == "BTC": + r = round(float(self._btc_to_usd(self.amount)), 5) + elif self.currency_symbol == "LBC": + r = round(float(self._btc_to_usd(self._lbc_to_btc(self.amount))), 5) + assert r is not None + return r + + def _usd_to_btc(self, usd): + return self._USDBTC['spot'] * float(usd) + + def _btc_to_usd(self, btc): + return float(btc) / self._USDBTC['spot'] + + def _btc_to_lbc(self, btc): + return float(btc) / self._BTCLBC['spot'] / (1.0 - BITTREX_FEE) + + def _lbc_to_btc(self, lbc): + return self._BTCLBC['spot'] * float(lbc) -def fee_from_dict(fee_dict): - s = fee_dict.keys()[0] - return LBRYFee(s, fee_dict[s]['amount'], fee_dict[s]['address']) +class LBRYFeeFormat(dict): + def __init__(self, fee_dict): + dict.__init__(self) + self.fee_version = None + f = fee_dict.copy() + assert len(fee_dict) == 1 + for currency in fee_dict: + assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) + self.update({currency: {}}) + + for version in FEE_REVISIONS: + for k in FEE_REVISIONS[version]['required']: + assert k in fee_dict, "Missing required fee field: %s" % k + self[currency].update({k: f.pop(k)}) + for k in FEE_REVISIONS[version]['optional']: + if k in fee_dict: + self[currency].update({k: f.pop(k)}) + if not len(f): + self.fee_version = version + break + assert f == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) class Metadata(dict): @@ -66,9 +108,6 @@ class Metadata(dict): self.metaversion = None m = metadata.copy() - if 'fee' in metadata: - assert fee_from_dict(metadata['fee']) - assert "sources" in metadata, "No sources given" for source in metadata['sources']: assert source in SOURCE_TYPES, "Unknown source type" @@ -78,9 +117,13 @@ class Metadata(dict): assert k in metadata, "Missing required metadata field: %s" % k self.update({k: m.pop(k)}) for k in METADATA_REVISIONS[version]['optional']: - if k in metadata: + if k == 'fee': + pass + elif k in metadata: self.update({k: m.pop(k)}) - if not len(m): + if not len(m) or m.keys() == ['fee']: self.metaversion = version break + if 'fee' in m: + self.update({'fee': LBRYFeeFormat(m.pop('fee'))}) assert m == {}, "Unknown metadata keys: %s" % json.dumps(m.keys()) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 278396186..e09bd020d 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -1,17 +1,12 @@ import sys -from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet -from lbrynet.core.client.ClientRequest import ClientRequest -from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError -from lbrynet.core.Error import InsufficientFundsError -from lbrynet.core.sqlite_helpers import rerun_if_locked -from lbrynet.conf import SOURCE_TYPES -from lbrynet.core.LBRYMetadata import Metadata - -from lbryum import SimpleConfig, Network -from lbryum.lbrycrd import COIN, TYPE_ADDRESS -from lbryum.wallet import WalletStorage, Wallet -from lbryum.commands import known_commands, Commands -from lbryum.transaction import Transaction +import datetime +import logging +import json +import subprocess +import socket +import time +import os +import requests from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from twisted.internet import threads, reactor, defer, task @@ -20,15 +15,24 @@ from twisted.enterprise import adbapi from collections import defaultdict, deque from zope.interface import implements from decimal import Decimal -import datetime -import logging -import json -import subprocess -import socket -import time -import os +from googlefinance import getQuotes + +from lbryum import SimpleConfig, Network +from lbryum.lbrycrd import COIN, TYPE_ADDRESS +from lbryum.wallet import WalletStorage, Wallet +from lbryum.commands import known_commands, Commands +from lbryum.transaction import Transaction + +from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet +from lbrynet.core.client.ClientRequest import ClientRequest +from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError +from lbrynet.core.Error import InsufficientFundsError +from lbrynet.core.sqlite_helpers import rerun_if_locked +from lbrynet.conf import SOURCE_TYPES +from lbrynet.core.LBRYMetadata import Metadata log = logging.getLogger(__name__) +log.setLevel(logging.INFO) alert = logging.getLogger("lbryalert." + __name__) @@ -80,6 +84,50 @@ class LBRYWallet(object): self._batch_count = 20 self._first_run = self._FIRST_RUN_UNKNOWN + self._USDBTC = None + self._BTCLBC = None + self._exchange_rate_updater = task.LoopingCall(self._update_exchange_rates) + + def _usd_to_btc(self): + if self._USDBTC is not None: + if int(time.time()) - int(self._USDBTC['ts']) < 600: + log.info("USDBTC quote is new enough") + return defer.succeed({}) + + log.info("Getting new USDBTC quote") + x = float(getQuotes('CURRENCY:USDBTC')[0]['LastTradePrice']) + return defer.succeed({'USDBTC': {'spot': x, 'ts': int(time.time())}}) + + def _btc_to_lbc(self): + if self._BTCLBC is not None: + if int(time.time()) - int(self._BTCLBC['ts']) < 600: + log.info("BTCLBC quote is new enough") + return defer.succeed({}) + + log.info("Getting new BTCLBC quote") + r = requests.get("https://bittrex.com/api/v1.1/public/getmarkethistory", {'market': 'BTC-LBC', 'count': 50}) + trades = json.loads(r.text)['result'] + vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades]) + x = (1.0 / float(vwap)) / 0.99975 + + return defer.succeed({'BTCLBC': {'spot': x, 'ts': int(time.time())}}) + + def _set_exchange_rates(self, rates): + if 'USDBTC' in rates: + assert int(time.time()) - int(rates['USDBTC']['ts']) < 3600, "new USDBTC quote is too old" + self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']} + log.info("Updated USDBTC rate: %s" % json.dumps(self._USDBTC)) + if 'BTCLBC' in rates: + assert int(time.time()) - int(rates['BTCLBC']['ts']) < 3600, "new BTCLBC quote is too old" + self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']} + log.info("Updated BTCLBC rate: %s" % json.dumps(self._BTCLBC)) + + def _update_exchange_rates(self): + d = self._usd_to_btc() + d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) + d.addCallback(lambda _: self._btc_to_lbc()) + d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) + def start(self): def start_manage(): @@ -87,6 +135,8 @@ class LBRYWallet(object): self.manage() return True + self._exchange_rate_updater.start(1800) + d = self._open_db() d.addCallback(lambda _: self._start()) d.addCallback(lambda _: start_manage()) diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 7ae0ac13f..7563f018b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -11,7 +11,7 @@ from twisted.internet.task import LoopingCall from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob -from lbrynet.core.LBRYMetadata import Metadata, fee_from_dict +from lbrynet.core.LBRYMetadata import Metadata, LBRYFee from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME @@ -90,9 +90,9 @@ class GetStream(object): self.stream_hash = self.metadata['sources']['lbry_sd_hash'] if 'fee' in self.metadata: - fee = fee_from_dict(self.metadata['fee']) - if fee.convert(amount_only=True) > self.max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name)) + self.fee = LBRYFee(self.metadata['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + if self.fee.to_lbc() > self.max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), self.max_key_fee, self.resolved_name)) return defer.fail(None) def _cause_timeout(): @@ -124,11 +124,12 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): if self.fee is not None: - reserved_points = self.wallet.reserve_points(self.fee.address, self.fee.convert(amount_only=True)) + x = self.fee.to_lbc() + reserved_points = self.wallet.reserve_points(self.fee.address, x) if reserved_points is None: return defer.fail(InsufficientFundsError()) - log.info("Key fee: %f --> %s" % (self.key_fee, self.key_fee_address)) - return self.wallet.send_points_to_address(reserved_points, self.key_fee) + log.info("Key fee: %f --> %s" % (x, self.fee.address)) + return self.wallet.send_points_to_address(reserved_points, self.fee.address) return defer.succeed(None) d = _pay_key_fee()