Merge pull request #963 from lbryio/redundant_exchange_apis

Added redundant market feed (continued)

CHANGELOG.md

@@ -26,7 +26,7 @@ at anytime.
   *
 
 ### Added
-  *
+  * Added redundant API server for currency conversion
   *
 
 ### Removed

lbrynet/daemon/ExchangeRateManager.py

@@ -5,7 +5,6 @@ import json
 from twisted.internet import defer, threads
 from twisted.internet.task import LoopingCall
 
-from lbrynet import conf
 from lbrynet.core.Error import InvalidExchangeRateResponse
 
 log = logging.getLogger(__name__)
@@ -45,11 +44,14 @@ class MarketFeed(object):
         self.fee = fee
         self.rate = None
         self._updater = LoopingCall(self._update_price)
+        self._online = True
 
-    @property
     def rate_is_initialized(self):
         return self.rate is not None
 
+    def is_online(self):
+        return self._online
+
     def _make_request(self):
         r = requests.get(self.url, self.params, timeout=self.REQUESTS_TIMEOUT)
         return r.text
@@ -62,19 +64,22 @@ class MarketFeed(object):
         return defer.succeed(from_amount / (1.0 - self.fee))
 
     def _save_price(self, price):
-        log.debug("Saving price update %f for %s" % (price, self.market))
+        log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name))
         self.rate = ExchangeRate(self.market, price, int(time.time()))
+        self._online = True
 
-    def _log_error(self, err):
+    def _on_error(self, err):
-        log.warning("There was a problem updating %s exchange rate information from %s\n%s",
+        log.warning(
+            "There was a problem updating %s exchange rate information from %s: %s",
             self.market, self.name, err)
+        self._online = False
 
     def _update_price(self):
         d = threads.deferToThread(self._make_request)
         d.addCallback(self._handle_response)
         d.addCallback(self._subtract_fee)
         d.addCallback(self._save_price)
-        d.addErrback(self._log_error)
+        d.addErrback(self._on_error)
         return d
 
     def start(self):
@@ -92,7 +97,7 @@ class BittrexFeed(MarketFeed):
             self,
             "BTCLBC",
             "Bittrex",
-            conf.settings['bittrex_feed'],
+            "https://bittrex.com/api/v1.1/public/getmarkethistory",
             {'market': 'BTC-LBC', 'count': 50},
             BITTREX_FEE
         )
@@ -151,24 +156,55 @@ class LBRYioBTCFeed(MarketFeed):
         return defer.succeed(1.0 / json_response['data']['btc_usd'])
 
 
-def get_default_market_feed(currency_pair):
+class CryptonatorBTCFeed(MarketFeed):
-    currencies = None
+    def __init__(self):
-    if isinstance(currency_pair, str):
+        MarketFeed.__init__(
-        currencies = (currency_pair[0:3], currency_pair[3:6])
+            self,
-    elif isinstance(currency_pair, tuple):
+            "USDBTC",
-        currencies = currency_pair
+            "cryptonator.com",
-    assert currencies is not None
+            "https://api.cryptonator.com/api/ticker/usd-btc",
+            {},
+            0.0,
+        )
 
-    if currencies == ("USD", "BTC"):
+    def _handle_response(self, response):
-        return LBRYioBTCFeed()
+        try:
-    elif currencies == ("BTC", "LBC"):
+            json_response = json.loads(response)
-        return LBRYioFeed()
+        except ValueError:
+            raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
+        if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
+                        'success' not in json_response or json_response['success'] is not True:
+            raise InvalidExchangeRateResponse(self.name, 'result not found')
+        return defer.succeed(float(json_response['ticker']['price']))
+
+
+
+class CryptonatorFeed(MarketFeed):
+    def __init__(self):
+        MarketFeed.__init__(
+            self,
+            "BTCLBC",
+            "cryptonator.com",
+            "https://api.cryptonator.com/api/ticker/btc-lbc",
+            {},
+            0.0,
+        )
+
+    def _handle_response(self, response):
+        try:
+            json_response = json.loads(response)
+        except ValueError:
+            raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
+        if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
+                        'success' not in json_response or json_response['success'] is not True:
+            raise InvalidExchangeRateResponse(self.name, 'result not found')
+        return defer.succeed(float(json_response['ticker']['price']))
 
 
 class ExchangeRateManager(object):
     def __init__(self):
         self.market_feeds = [
-            get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS]
+           LBRYioBTCFeed(), LBRYioFeed(), BittrexFeed(), CryptonatorBTCFeed(), CryptonatorFeed()]
 
     def start(self):
         log.info("Starting exchange rate manager")
@@ -185,12 +221,13 @@ class ExchangeRateManager(object):
         log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
         if from_currency == to_currency:
             return amount
+
         for market in self.market_feeds:
-            if (market.rate_is_initialized and
+            if (market.rate_is_initialized() and market.is_online() and
                 market.rate.currency_pair == (from_currency, to_currency)):
                 return amount * market.rate.spot
         for market in self.market_feeds:
-            if (market.rate_is_initialized and
+            if (market.rate_is_initialized() and market.is_online() and
                 market.rate.currency_pair[0] == from_currency):
                 return self.convert_currency(
                     market.rate.currency_pair[1], to_currency, amount * market.rate.spot)

lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py

@@ -119,3 +119,68 @@ class LBRYioBTCFeedTest(unittest.TestCase):
         response = '{"success":true,"result":[]}'
         with self.assertRaises(InvalidExchangeRateResponse):
             out = yield feed._handle_response(response)
+
+class CryptonatorFeedTest(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_handle_response(self):
+        feed = ExchangeRateManager.CryptonatorFeed()
+
+        response = '{\"ticker\":{\"base\":\"BTC\",\"target\":\"LBC\",\"price\":\"23657.44026496\"' \
+                   ',\"volume\":\"\",\"change\":\"-5.59806916\"},\"timestamp\":1507470422' \
+                   ',\"success\":true,\"error\":\"\"}'
+        out = yield feed._handle_response(response)
+        expected = 23657.44026496
+        self.assertEqual(expected, out)
+
+        response = '{}'
+        with self.assertRaises(InvalidExchangeRateResponse):
+            out = yield feed._handle_response(response)
+
+        response = '{"success":true,"ticker":{}}'
+        with self.assertRaises(InvalidExchangeRateResponse):
+            out = yield feed._handle_response(response)
+
+class CryptonatorBTCFeedTest(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_handle_response(self):
+        feed = ExchangeRateManager.CryptonatorBTCFeed()
+
+        response = '{\"ticker\":{\"base\":\"USD\",\"target\":\"BTC\",\"price\":\"0.00022123\",' \
+                   '\"volume\":\"\",\"change\":\"-0.00000259\"},\"timestamp\":1507471141,' \
+                   '\"success\":true,\"error\":\"\"}'
+        out = yield feed._handle_response(response)
+        expected = 0.00022123
+        self.assertEqual(expected, out)
+
+        response = '{}'
+        with self.assertRaises(InvalidExchangeRateResponse):
+            out = yield feed._handle_response(response)
+
+        response = '{"success":true,"ticker":{}}'
+        with self.assertRaises(InvalidExchangeRateResponse):
+            out = yield feed._handle_response(response)
+
+
+class BittrexFeedTest(unittest.TestCase):
+
+    @defer.inlineCallbacks
+    def test_handle_response(self):
+        feed = ExchangeRateManager.BittrexFeed()
+
+        response = '{"success":true,"message":"","result":[{"Id":6902471,"TimeStamp":"2017-02-2'\
+        '7T23:41:52.213","Quantity":56.12611239,"Price":0.00001621,"Total":0.00090980,"FillType":"'\
+        'PARTIAL_FILL","OrderType":"SELL"},{"Id":6902403,"TimeStamp":"2017-02-27T23:31:40.463","Qu'\
+        'antity":430.99988180,"Price":0.00001592,"Total":0.00686151,"FillType":"PARTIAL_FILL","Ord'\
+        'erType":"SELL"}]}'
+        out = yield feed._handle_response(response)
+        expected = 1.0 / ((0.00090980+0.00686151) / (56.12611239+430.99988180))
+        self.assertEqual(expected, out)
+
+        response = '{}'
+        with self.assertRaises(InvalidExchangeRateResponse):
+            out = yield feed._handle_response(response)
+
+        response = '{"success":true,"result":[]}'
+        with self.assertRaises(InvalidExchangeRateResponse):
+            out = yield feed._handle_response(response)
+