From 415495fc1688a41c763023ef0da4808241ae78ee Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 28 Jul 2016 05:30:13 -0400 Subject: [PATCH] LBRYExchangeRateManager --- lbrynet/core/LBRYMetadata.py | 48 +----- lbrynet/core/LBRYWallet.py | 49 ------ lbrynet/lbrynet_daemon/LBRYDaemon.py | 8 +- lbrynet/lbrynet_daemon/LBRYDownloader.py | 23 +-- .../lbrynet_daemon/LBRYExchangeRateManager.py | 150 ++++++++++++++++++ 5 files changed, 170 insertions(+), 108 deletions(-) create mode 100644 lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 9def7f8cd..2bf31c9c7 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -32,6 +32,7 @@ class LBRYFeeValidator(dict): dict.__init__(self) assert len(fee_dict) == 1 self.fee_version = None + self.currency_symbol = None fee_to_load = deepcopy(fee_dict) @@ -68,53 +69,6 @@ class LBRYFeeValidator(dict): self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) -class LBRYFee(LBRYFeeValidator): - def __init__(self, fee_dict, rate_dict, bittrex_fee=None): - LBRYFeeValidator.__init__(self, fee_dict) - self.bittrex_fee = BITTREX_FEE if bittrex_fee is None else bittrex_fee - rates = deepcopy(rate_dict) - - assert 'BTCLBC' in rates and 'USDBTC' in rates - for fx in rate_dict: - assert int(time.time()) - int(rates[fx]['ts']) < 3600, "%s quote is out of date" % fx - self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']} - self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']} - - def to_lbc(self): - r = None - if self.currency_symbol == "LBC": - r = round(float(self.amount), 5) - elif self.currency_symbol == "BTC": - r = round(float(self._btc_to_lbc(self.amount)), 5) - elif self.currency_symbol == "USD": - r = round(float(self._btc_to_lbc(self._usd_to_btc(self.amount))), 5) - assert r is not None - return r - - def to_usd(self): - r = None - if self.currency_symbol == "USD": - r = round(float(self.amount), 5) - elif self.currency_symbol == "BTC": - r = round(float(self._btc_to_usd(self.amount)), 5) - elif self.currency_symbol == "LBC": - r = round(float(self._btc_to_usd(self._lbc_to_btc(self.amount))), 5) - assert r is not None - return r - - def _usd_to_btc(self, usd): - return self._USDBTC['spot'] * float(usd) - - def _btc_to_usd(self, btc): - return float(btc) / self._USDBTC['spot'] - - def _btc_to_lbc(self, btc): - return float(btc) * self._BTCLBC['spot'] / (1.0 - self.bittrex_fee) - - def _lbc_to_btc(self, lbc): - return self._BTCLBC['spot'] / float(lbc) - - class Metadata(dict): def __init__(self, metadata): dict.__init__(self) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 1e567169d..7d7ab8db7 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -82,50 +82,6 @@ class LBRYWallet(object): self._batch_count = 20 self._first_run = self._FIRST_RUN_UNKNOWN - self._USDBTC = None - self._BTCLBC = None - self._exchange_rate_updater = task.LoopingCall(self._update_exchange_rates) - - def _usd_to_btc(self): - if self._USDBTC is not None: - if int(time.time()) - int(self._USDBTC['ts']) < 600: - log.info("USDBTC quote is new enough") - return defer.succeed({}) - - log.info("Getting new USDBTC quote") - x = float(getQuotes('CURRENCY:USDBTC')[0]['LastTradePrice']) - return defer.succeed({'USDBTC': {'spot': x, 'ts': int(time.time())}}) - - def _btc_to_lbc(self): - if self._BTCLBC is not None: - if int(time.time()) - int(self._BTCLBC['ts']) < 600: - log.info("BTCLBC quote is new enough") - return defer.succeed({}) - - log.info("Getting new BTCLBC quote") - r = requests.get("https://bittrex.com/api/v1.1/public/getmarkethistory", {'market': 'BTC-LBC', 'count': 50}) - trades = json.loads(r.text)['result'] - vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades]) - x = (1.0 / float(vwap)) / 0.99975 - - return defer.succeed({'BTCLBC': {'spot': x, 'ts': int(time.time())}}) - - def _set_exchange_rates(self, rates): - if 'USDBTC' in rates: - assert int(time.time()) - int(rates['USDBTC']['ts']) < 3600, "new USDBTC quote is too old" - self._USDBTC = {'spot': rates['USDBTC']['spot'], 'ts': rates['USDBTC']['ts']} - log.info("Updated USDBTC rate: %s" % json.dumps(self._USDBTC)) - if 'BTCLBC' in rates: - assert int(time.time()) - int(rates['BTCLBC']['ts']) < 3600, "new BTCLBC quote is too old" - self._BTCLBC = {'spot': rates['BTCLBC']['spot'], 'ts': rates['BTCLBC']['ts']} - log.info("Updated BTCLBC rate: %s" % json.dumps(self._BTCLBC)) - - def _update_exchange_rates(self): - d = self._usd_to_btc() - d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) - d.addCallback(lambda _: self._btc_to_lbc()) - d.addCallbacks(self._set_exchange_rates, lambda _: reactor.callLater(30, self._update_exchange_rates)) - def start(self): def start_manage(): @@ -133,8 +89,6 @@ class LBRYWallet(object): self.manage() return True - self._exchange_rate_updater.start(1800) - d = self._open_db() d.addCallback(lambda _: self._start()) d.addCallback(lambda _: start_manage()) @@ -147,9 +101,6 @@ class LBRYWallet(object): def stop(self): self.stopped = True - - if self._exchange_rate_updater.running: - self._exchange_rate_updater.stop() # If self.next_manage_call is None, then manage is currently running or else # start has not been called, so set stopped and do nothing else. diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 601c6ae0f..89b07e504 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -41,6 +41,7 @@ from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifi from lbrynet.lbrynet_daemon.LBRYUIManager import LBRYUIManager from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher +from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.core import utils from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings @@ -157,6 +158,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.current_db_revision = 1 self.run_server = True self.session = None + self.exchange_rate_manager = ExchangeRateManager() self.waiting_on = {} self.streams = {} self.pending_claims = {} @@ -484,6 +486,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) self.connection_problem_checker.start(1) + self.exchange_rate_manager.start() + if host_ui: self.lbry_ui_manager.update_checker.start(1800, now=False) @@ -1122,8 +1126,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet, - self.lbry_file_manager, max_key_fee=self.max_key_fee, - data_rate=self.data_rate, timeout=timeout, + self.lbry_file_manager, self.exchange_rate_manager, + max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout, download_directory=download_directory, file_name=file_name) d = self.streams[name].start(stream_info, name) if wait_for_write: diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 8106b1fd4..62d9cd33d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -12,7 +12,7 @@ from twisted.internet.task import LoopingCall from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob -from lbrynet.core.LBRYMetadata import Metadata, LBRYFee +from lbrynet.core.LBRYMetadata import Metadata, LBRYFeeValidator from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME @@ -42,8 +42,8 @@ log = logging.getLogger(__name__) class GetStream(object): - def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, data_rate=0.5, - timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None): + def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager, + max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None): self.wallet = wallet self.resolved_name = None self.description = None @@ -52,6 +52,7 @@ class GetStream(object): self.name = None self.file_name = file_name self.session = session + self.exchange_rate_manager = exchange_rate_manager self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager) self.lbry_file_manager = lbry_file_manager self.sd_identifier = sd_identifier @@ -86,8 +87,10 @@ class GetStream(object): def _convert_max_fee(self): if isinstance(self.max_key_fee, dict): - max_fee = deepcopy(self.max_key_fee) - return LBRYFee(max_fee, {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}).to_lbc() + max_fee = LBRYFeeValidator(self.max_key_fee) + if max_fee.currency_symbol == "LBC": + return max_fee.amount + return self.exchange_rate_manager.to_lbc(self.fee).amount elif isinstance(self.max_key_fee, float): return float(self.max_key_fee) @@ -122,14 +125,14 @@ class GetStream(object): self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] if 'fee' in self.stream_info: - self.fee = LBRYFee(self.stream_info['fee'], {'USDBTC': self.wallet._USDBTC, 'BTCLBC': self.wallet._BTCLBC}) + self.fee = LBRYFeeValidator(self.stream_info['fee']) max_key_fee = self._convert_max_fee() - if self.fee.to_lbc() > max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.to_lbc(), + if self.exchange_rate_manager.to_lbc(self.fee).amount > max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount, self.max_key_fee, self.resolved_name)) return defer.fail(KeyFeeAboveMaxAllowed()) - log.info("Key fee %f below limit of %f, downloading lbry://%s" % (self.fee.to_lbc(), + log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee), max_key_fee, self.resolved_name)) @@ -149,7 +152,7 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): if self.fee is not None: - fee_lbc = float(self.fee.to_lbc()) + fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) if reserved_points is None: return defer.fail(InsufficientFundsError()) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py new file mode 100644 index 000000000..2db0d526a --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -0,0 +1,150 @@ +import time +import requests +import logging +import json +import googlefinance +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall + +from lbrynet.core.LBRYMetadata import LBRYFeeValidator + +log = logging.getLogger(__name__) + +CURRENCY_PAIRS = ["USDBTC", "BTCLBC"] +BITTREX_FEE = 0.0025 +COINBASE_FEE = 0.0 #add fee + + +class ExchangeRate(object): + def __init__(self, market, spot, ts): + assert int(time.time()) - ts < 600 + self.currency_pair = (market[0:3], market[3:6]) + self.spot = spot + self.ts = ts + + def as_dict(self): + return {'spot': self.spot, 'ts': self.ts} + + +class MarketFeed(object): + def __init__(self, market, name, url, params, fee): + self.market = market + self.name = name + self.url = url + self.params = params + self.fee = fee + self.rate = None + self._updater = LoopingCall(self._update_price) + + def _make_request(self): + r = requests.get(self.url, self.params) + return r.text + + def _handle_response(self, response): + return NotImplementedError + + def _subtract_fee(self, from_amount): + return defer.succeed(from_amount / (1.0 - self.fee)) + + def _save_price(self, price): + log.info("Saving price update %f for %s" % (price, self.market)) + self.rate = ExchangeRate(self.market, price, int(time.time())) + + def _update_price(self): + d = defer.succeed(self._make_request()) + d.addCallback(self._handle_response) + d.addCallback(self._subtract_fee) + d.addCallback(self._save_price) + + def start(self): + if not self._updater.running: + self._updater.start(15) + + def stop(self): + if self._updater.running: + self._updater.stop() + + +class BittrexFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "BTCLBC", + "Bittrex", + "https://bittrex.com/api/v1.1/public/getmarkethistory", + {'market': 'BTC-LBC', 'count': 50}, + BITTREX_FEE + ) + + def _handle_response(self, response): + trades = json.loads(response)['result'] + vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades]) + return defer.succeed(float(1.0 / vwap)) + + +class GoogleBTCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "USDBTC", + "Coinbase via Google finance", + None, + None, + COINBASE_FEE + ) + + def _make_request(self): + return googlefinance.getQuotes('CURRENCY:USDBTC')[0] + + def _handle_response(self, response): + return float(response['LastTradePrice']) + + +def get_default_market_feed(currency_pair): + currencies = None + if isinstance(currency_pair, str): + currencies = (currency_pair[0:3], currency_pair[3:6]) + elif isinstance(currency_pair, tuple): + currencies = currency_pair + assert currencies is not None + + if currencies == ("USD", "BTC"): + return GoogleBTCFeed() + elif currencies == ("BTC", "LBC"): + return BittrexFeed() + + +class ExchangeRateManager(object): + def __init__(self): + reactor.addSystemEventTrigger('before', 'shutdown', self.stop) + self.market_feeds = [get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS] + + def start(self): + log.info("Starting exchange rate manager") + for feed in self.market_feeds: + feed.start() + + def stop(self): + log.info("Stopping exchange rate manager") + for source in self.market_feeds: + source.stop() + + def convert_currency(self, from_currency, to_currency, amount): + log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) + for market in self.market_feeds: + if market.rate.currency_pair == (from_currency, to_currency): + return amount * market.rate.spot + for market in self.market_feeds: + if market.rate.currency_pair[0] == from_currency: + return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) + + def fee_dict(self): + return {market: market.rate.as_dict() for market in self.market_feeds} + + def to_lbc(self, fee): + return LBRYFeeValidator({fee.currency_symbol: + { + 'amount': self.convert_currency(fee.currency_symbol, "LBC", fee.amount), + 'address': fee.address + } + })