From aa15f6c4b83930097272dbe0d94f0860fc30afb7 Mon Sep 17 00:00:00 2001 From: Miroslav Kovar Date: Thu, 21 Sep 2017 17:49:01 +0200 Subject: [PATCH 01/17] Added redundant market feed (cryptonator). --- lbrynet/daemon/ExchangeRateManager.py | 60 ++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index 63298ed88..40020fa7c 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -45,14 +45,23 @@ class MarketFeed(object): self.fee = fee self.rate = None self._updater = LoopingCall(self._update_price) + self._online = True @property def rate_is_initialized(self): return self.rate is not None + @property + def is_online(self): + return self._online + def _make_request(self): r = requests.get(self.url, self.params, timeout=self.REQUESTS_TIMEOUT) - return r.text + if r.status_code == 200: + self._online = True + return r.text + self._online = False + return "" def _handle_response(self, response): return NotImplementedError @@ -66,7 +75,8 @@ class MarketFeed(object): self.rate = ExchangeRate(self.market, price, int(time.time())) def _log_error(self, err): - log.warning("There was a problem updating %s exchange rate information from %s\n%s", + log.warning( + "There was a problem updating %s exchange rate information from %s: %s", self.market, self.name, err) def _update_price(self): @@ -151,6 +161,45 @@ class LBRYioBTCFeed(MarketFeed): return defer.succeed(1.0 / json_response['data']['btc_usd']) +class CryptonatorBTCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "USDBTC", + "cryptonator.com", + "https://api.cryptonator.com/api/ticker/usd-btc", + {}, + 0.0, + ) + + def _handle_response(self, response): + json_response = json.loads(response) + if 'ticker' not in json_response or 'success' not in json_response or \ + json_response['success'] is not True: + raise InvalidExchangeRateResponse(self.name, 'result not found') + return defer.succeed(float(json_response['ticker']['price'])) + + + +class CryptonatorFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "BTCLBC", + "cryptonator.com", + "https://api.cryptonator.com/api/ticker/btc-lbc", + {}, + 0.0, + ) + + def _handle_response(self, response): + json_response = json.loads(response) + if 'ticker' not in json_response or 'success' not in json_response or \ + json_response['success'] is not True: + raise InvalidExchangeRateResponse(self.name, 'result not found') + return defer.succeed(float(json_response['ticker']['price'])) + + def get_default_market_feed(currency_pair): currencies = None if isinstance(currency_pair, str): @@ -168,7 +217,7 @@ def get_default_market_feed(currency_pair): class ExchangeRateManager(object): def __init__(self): self.market_feeds = [ - get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS] + LBRYioBTCFeed(), LBRYioFeed(), CryptonatorBTCFeed(), CryptonatorFeed()] def start(self): log.info("Starting exchange rate manager") @@ -185,12 +234,13 @@ class ExchangeRateManager(object): log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates)) if from_currency == to_currency: return amount + for market in self.market_feeds: - if (market.rate_is_initialized and + if (market.rate_is_initialized and market.is_online and market.rate.currency_pair == (from_currency, to_currency)): return amount * market.rate.spot for market in self.market_feeds: - if (market.rate_is_initialized and + if (market.rate_is_initialized and market.is_online and market.rate.currency_pair[0] == from_currency): return self.convert_currency( market.rate.currency_pair[1], to_currency, amount * market.rate.spot) From 34643b7f95b0a3fd0cebb09979e1e3d06167c2bc Mon Sep 17 00:00:00 2001 From: Miroslav Kovar Date: Thu, 21 Sep 2017 23:36:07 +0200 Subject: [PATCH 02/17] Added changelog entry. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90024ba39..820e64931 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ at anytime. * ### Added - * + * Added redundant API server for currency conversion * ### Removed From 433547407e6cac983bad24e5691e4f48027fe722 Mon Sep 17 00:00:00 2001 From: Miroslav Kovar Date: Sun, 8 Oct 2017 16:15:53 +0200 Subject: [PATCH 03/17] Added tests. --- lbrynet/daemon/ExchangeRateManager.py | 18 ++++++--- .../test_ExchangeRateManager.py | 40 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index 40020fa7c..3e94b8e0f 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -173,9 +173,12 @@ class CryptonatorBTCFeed(MarketFeed): ) def _handle_response(self, response): - json_response = json.loads(response) - if 'ticker' not in json_response or 'success' not in json_response or \ - json_response['success'] is not True: + try: + json_response = json.loads(response) + except ValueError: + raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response) + if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \ + 'success' not in json_response or json_response['success'] is not True: raise InvalidExchangeRateResponse(self.name, 'result not found') return defer.succeed(float(json_response['ticker']['price'])) @@ -193,9 +196,12 @@ class CryptonatorFeed(MarketFeed): ) def _handle_response(self, response): - json_response = json.loads(response) - if 'ticker' not in json_response or 'success' not in json_response or \ - json_response['success'] is not True: + try: + json_response = json.loads(response) + except ValueError: + raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response) + if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \ + 'success' not in json_response or json_response['success'] is not True: raise InvalidExchangeRateResponse(self.name, 'result not found') return defer.succeed(float(json_response['ticker']['price'])) diff --git a/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py b/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py index 9c82b449b..0c843209b 100644 --- a/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py +++ b/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py @@ -119,3 +119,43 @@ class LBRYioBTCFeedTest(unittest.TestCase): response = '{"success":true,"result":[]}' with self.assertRaises(InvalidExchangeRateResponse): out = yield feed._handle_response(response) + +class CryptonatorFeedTest(unittest.TestCase): + @defer.inlineCallbacks + def test_handle_response(self): + feed = ExchangeRateManager.CryptonatorFeed() + + response = '{\"ticker\":{\"base\":\"BTC\",\"target\":\"LBC\",\"price\":\"23657.44026496\"' \ + ',\"volume\":\"\",\"change\":\"-5.59806916\"},\"timestamp\":1507470422' \ + ',\"success\":true,\"error\":\"\"}' + out = yield feed._handle_response(response) + expected = 23657.44026496 + self.assertEqual(expected, out) + + response = '{}' + with self.assertRaises(InvalidExchangeRateResponse): + out = yield feed._handle_response(response) + + response = '{"success":true,"ticker":{}}' + with self.assertRaises(InvalidExchangeRateResponse): + out = yield feed._handle_response(response) + +class CryptonatorBTCFeedTest(unittest.TestCase): + @defer.inlineCallbacks + def test_handle_response(self): + feed = ExchangeRateManager.CryptonatorBTCFeed() + + response = '{\"ticker\":{\"base\":\"USD\",\"target\":\"BTC\",\"price\":\"0.00022123\",' \ + '\"volume\":\"\",\"change\":\"-0.00000259\"},\"timestamp\":1507471141,' \ + '\"success\":true,\"error\":\"\"}' + out = yield feed._handle_response(response) + expected = 0.00022123 + self.assertEqual(expected, out) + + response = '{}' + with self.assertRaises(InvalidExchangeRateResponse): + out = yield feed._handle_response(response) + + response = '{"success":true,"ticker":{}}' + with self.assertRaises(InvalidExchangeRateResponse): + out = yield feed._handle_response(response) From 4411c5c202e8db4f5e80aafd318d0b1c2c934625 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 23 Oct 2017 16:04:10 -0400 Subject: [PATCH 04/17] add name of feed to log.info --- lbrynet/daemon/ExchangeRateManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index 3e94b8e0f..de761a85c 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -71,7 +71,7 @@ class MarketFeed(object): return defer.succeed(from_amount / (1.0 - self.fee)) def _save_price(self, price): - log.debug("Saving price update %f for %s" % (price, self.market)) + log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name)) self.rate = ExchangeRate(self.market, price, int(time.time())) def _log_error(self, err): From 7e7931fbf1e25dcab60634ba2ae008f9195b950b Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 23 Oct 2017 16:05:41 -0400 Subject: [PATCH 05/17] add Bittrex feed as a redundant feed, add unit test for it --- lbrynet/daemon/ExchangeRateManager.py | 5 ++-- .../test_ExchangeRateManager.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index de761a85c..dfe02bbc1 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -5,7 +5,6 @@ import json from twisted.internet import defer, threads from twisted.internet.task import LoopingCall -from lbrynet import conf from lbrynet.core.Error import InvalidExchangeRateResponse log = logging.getLogger(__name__) @@ -102,7 +101,7 @@ class BittrexFeed(MarketFeed): self, "BTCLBC", "Bittrex", - conf.settings['bittrex_feed'], + "https://bittrex.com/api/v1.1/public/getmarkethistory", {'market': 'BTC-LBC', 'count': 50}, BITTREX_FEE ) @@ -223,7 +222,7 @@ def get_default_market_feed(currency_pair): class ExchangeRateManager(object): def __init__(self): self.market_feeds = [ - LBRYioBTCFeed(), LBRYioFeed(), CryptonatorBTCFeed(), CryptonatorFeed()] + LBRYioBTCFeed(), LBRYioFeed(), BittrexFeed(), CryptonatorBTCFeed(), CryptonatorFeed()] def start(self): log.info("Starting exchange rate manager") diff --git a/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py b/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py index 0c843209b..a7d4cb599 100644 --- a/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py +++ b/lbrynet/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py @@ -159,3 +159,28 @@ class CryptonatorBTCFeedTest(unittest.TestCase): response = '{"success":true,"ticker":{}}' with self.assertRaises(InvalidExchangeRateResponse): out = yield feed._handle_response(response) + + +class BittrexFeedTest(unittest.TestCase): + + @defer.inlineCallbacks + def test_handle_response(self): + feed = ExchangeRateManager.BittrexFeed() + + response = '{"success":true,"message":"","result":[{"Id":6902471,"TimeStamp":"2017-02-2'\ + '7T23:41:52.213","Quantity":56.12611239,"Price":0.00001621,"Total":0.00090980,"FillType":"'\ + 'PARTIAL_FILL","OrderType":"SELL"},{"Id":6902403,"TimeStamp":"2017-02-27T23:31:40.463","Qu'\ + 'antity":430.99988180,"Price":0.00001592,"Total":0.00686151,"FillType":"PARTIAL_FILL","Ord'\ + 'erType":"SELL"}]}' + out = yield feed._handle_response(response) + expected = 1.0 / ((0.00090980+0.00686151) / (56.12611239+430.99988180)) + self.assertEqual(expected, out) + + response = '{}' + with self.assertRaises(InvalidExchangeRateResponse): + out = yield feed._handle_response(response) + + response = '{"success":true,"result":[]}' + with self.assertRaises(InvalidExchangeRateResponse): + out = yield feed._handle_response(response) + From 020a2e0e8995155e4d9d4d3ccf32fe7f0da88bad Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 23 Oct 2017 16:44:26 -0400 Subject: [PATCH 06/17] adjust the _online variable to be set on the errback, so any exception will set _online to False --- lbrynet/daemon/ExchangeRateManager.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index dfe02bbc1..339f09383 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -56,11 +56,7 @@ class MarketFeed(object): def _make_request(self): r = requests.get(self.url, self.params, timeout=self.REQUESTS_TIMEOUT) - if r.status_code == 200: - self._online = True - return r.text - self._online = False - return "" + return r.text def _handle_response(self, response): return NotImplementedError @@ -72,18 +68,20 @@ class MarketFeed(object): def _save_price(self, price): log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name)) self.rate = ExchangeRate(self.market, price, int(time.time())) + self._online = True - def _log_error(self, err): + def _on_error(self, err): log.warning( "There was a problem updating %s exchange rate information from %s: %s", self.market, self.name, err) + self._online = False def _update_price(self): d = threads.deferToThread(self._make_request) d.addCallback(self._handle_response) d.addCallback(self._subtract_fee) d.addCallback(self._save_price) - d.addErrback(self._log_error) + d.addErrback(self._on_error) return d def start(self): From cf9f9d0e7520ded8c946c07af7ca5db9a015f08c Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 23 Oct 2017 16:44:40 -0400 Subject: [PATCH 07/17] remove unused get_default_market_feed() function --- lbrynet/daemon/ExchangeRateManager.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index 339f09383..25a06e420 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -203,20 +203,6 @@ class CryptonatorFeed(MarketFeed): return defer.succeed(float(json_response['ticker']['price'])) -def get_default_market_feed(currency_pair): - currencies = None - if isinstance(currency_pair, str): - currencies = (currency_pair[0:3], currency_pair[3:6]) - elif isinstance(currency_pair, tuple): - currencies = currency_pair - assert currencies is not None - - if currencies == ("USD", "BTC"): - return LBRYioBTCFeed() - elif currencies == ("BTC", "LBC"): - return LBRYioFeed() - - class ExchangeRateManager(object): def __init__(self): self.market_feeds = [ From ea4b6935c99227e8e8ce1ef41ee920ac8ae1b6c7 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 30 Oct 2017 16:01:25 -0400 Subject: [PATCH 08/17] remove unnecessary use of @property --- lbrynet/daemon/ExchangeRateManager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lbrynet/daemon/ExchangeRateManager.py b/lbrynet/daemon/ExchangeRateManager.py index 25a06e420..923eb4b2b 100644 --- a/lbrynet/daemon/ExchangeRateManager.py +++ b/lbrynet/daemon/ExchangeRateManager.py @@ -46,11 +46,9 @@ class MarketFeed(object): self._updater = LoopingCall(self._update_price) self._online = True - @property def rate_is_initialized(self): return self.rate is not None - @property def is_online(self): return self._online @@ -225,11 +223,11 @@ class ExchangeRateManager(object): return amount for market in self.market_feeds: - if (market.rate_is_initialized and market.is_online and + if (market.rate_is_initialized() and market.is_online() and market.rate.currency_pair == (from_currency, to_currency)): return amount * market.rate.spot for market in self.market_feeds: - if (market.rate_is_initialized and market.is_online and + if (market.rate_is_initialized() and market.is_online() and market.rate.currency_pair[0] == from_currency): return self.convert_currency( market.rate.currency_pair[1], to_currency, amount * market.rate.spot) From d68ca65e41d2aa76fafbd9031df269739dc0a120 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Fri, 29 Sep 2017 14:29:35 -0400 Subject: [PATCH 09/17] warn if reader is garbage collected but not closed, do the same for writer --- lbrynet/blob/reader.py | 6 +++++- lbrynet/blob/writer.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lbrynet/blob/reader.py b/lbrynet/blob/reader.py index dd248c8fd..4d1f51dff 100644 --- a/lbrynet/blob/reader.py +++ b/lbrynet/blob/reader.py @@ -40,11 +40,15 @@ class HashBlobReader(object): read(size) and close() """ def __init__(self, file_path, finished_cb): + self.file_path = file_path self.finished_cb = finished_cb self.finished_cb_d = None - self.read_handle = open(file_path, 'rb') + self.read_handle = open(self.file_path, 'rb') def __del__(self): + if self.finished_cb_d is None: + log.warn("Garbage collection was called, but reader for %s was not closed yet", + self.file_path) self.close() def read(self, size=-1): diff --git a/lbrynet/blob/writer.py b/lbrynet/blob/writer.py index a95430386..dc4d3d77a 100644 --- a/lbrynet/blob/writer.py +++ b/lbrynet/blob/writer.py @@ -16,6 +16,11 @@ class HashBlobWriter(object): self._hashsum = get_lbry_hash_obj() self.len_so_far = 0 + def __del__(self): + if self.finished_cb_d is None: + log.warn("Garbage collection was called, but writer was not closed yet") + self.close() + @property def blob_hash(self): return self._hashsum.hexdigest() From e07c2f7bd8d7152a592b614c9ccbc6a2e48d5ae0 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 2 Oct 2017 18:07:07 -0400 Subject: [PATCH 10/17] take read handle as argument instead of file path --- lbrynet/blob/blob_file.py | 3 ++- lbrynet/blob/reader.py | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lbrynet/blob/blob_file.py b/lbrynet/blob/blob_file.py index 78cf974ad..0c4d2555c 100644 --- a/lbrynet/blob/blob_file.py +++ b/lbrynet/blob/blob_file.py @@ -78,7 +78,8 @@ class BlobFile(object): finished """ if self._verified is True: - reader = HashBlobReader(self.file_path, self.reader_finished) + f = open(self.file_path, 'rb') + reader = HashBlobReader(f, self.reader_finished) self.readers += 1 return reader return None diff --git a/lbrynet/blob/reader.py b/lbrynet/blob/reader.py index 4d1f51dff..745e62ef8 100644 --- a/lbrynet/blob/reader.py +++ b/lbrynet/blob/reader.py @@ -39,16 +39,15 @@ class HashBlobReader(object): This is a file like reader class that supports read(size) and close() """ - def __init__(self, file_path, finished_cb): - self.file_path = file_path + def __init__(self, read_handle, finished_cb): self.finished_cb = finished_cb self.finished_cb_d = None - self.read_handle = open(self.file_path, 'rb') + self.read_handle = read_handle def __del__(self): if self.finished_cb_d is None: log.warn("Garbage collection was called, but reader for %s was not closed yet", - self.file_path) + self.read_handle.name) self.close() def read(self, size=-1): From 136034539514d5d56f5c7539a5ebccd9033a99f5 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Fri, 29 Sep 2017 14:43:03 -0400 Subject: [PATCH 11/17] use blob.open_for_reading() in StreamBlobDecryptor --- lbrynet/cryptstream/CryptBlob.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lbrynet/cryptstream/CryptBlob.py b/lbrynet/cryptstream/CryptBlob.py index a7303a588..6d4bcbc5e 100644 --- a/lbrynet/cryptstream/CryptBlob.py +++ b/lbrynet/cryptstream/CryptBlob.py @@ -1,6 +1,6 @@ import binascii import logging -from twisted.internet import defer +from twisted.internet import defer, threads from cryptography.hazmat.primitives.ciphers import Cipher, modes from cryptography.hazmat.primitives.ciphers.algorithms import AES from cryptography.hazmat.primitives.padding import PKCS7 @@ -46,6 +46,10 @@ class StreamBlobDecryptor(object): write_func - function that takes decrypted string as arugment and writes it somewhere + + Returns: + + deferred that returns after decrypting blob and writing content """ def remove_padding(data): @@ -67,13 +71,17 @@ class StreamBlobDecryptor(object): last_chunk = self.cipher.update(data_to_decrypt) + self.cipher.finalize() write_func(remove_padding(last_chunk)) - def decrypt_bytes(data): + + read_handle = self.blob.open_for_reading() + + def decrypt_bytes(): + data = read_handle.read() self.buff += data self.len_read += len(data) write_bytes() + finish_decrypt() - d = self.blob.read(decrypt_bytes) - d.addCallback(lambda _: finish_decrypt()) + d = threads.deferToThread(decrypt_bytes) return d From 96d8cb17d91df4662fec9f648bc8a5a688d00dc9 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Fri, 29 Sep 2017 14:45:11 -0400 Subject: [PATCH 12/17] delete deprecated producer/consumer read methods from BlobFile --- lbrynet/blob/blob_file.py | 30 +----------------------------- lbrynet/blob/reader.py | 31 ------------------------------- 2 files changed, 1 insertion(+), 60 deletions(-) diff --git a/lbrynet/blob/blob_file.py b/lbrynet/blob/blob_file.py index 0c4d2555c..5c08b2059 100644 --- a/lbrynet/blob/blob_file.py +++ b/lbrynet/blob/blob_file.py @@ -1,14 +1,13 @@ import logging import os from twisted.internet import defer, threads -from twisted.protocols.basic import FileSender from twisted.web.client import FileBodyProducer from twisted.python.failure import Failure from lbrynet import conf from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError from lbrynet.core.utils import is_valid_blobhash from lbrynet.blob.writer import HashBlobWriter -from lbrynet.blob.reader import HashBlobReader, HashBlobReader_v0 +from lbrynet.blob.reader import HashBlobReader log = logging.getLogger(__name__) @@ -143,33 +142,6 @@ class BlobFile(object): return True return False - def read(self, write_func): - """ - This function is only used in StreamBlobDecryptor - and should be deprecated in favor of open_for_reading() - """ - def close_self(*args): - self.close_read_handle(file_handle) - return args[0] - - file_sender = FileSender() - reader = HashBlobReader_v0(write_func) - file_handle = self.open_for_reading() - if file_handle is not None: - d = file_sender.beginFileTransfer(file_handle, reader) - d.addCallback(close_self) - else: - d = defer.fail(IOError("Could not read the blob")) - return d - - def close_read_handle(self, file_handle): - """ - This function is only used in StreamBlobDecryptor - and should be deprecated in favor of open_for_reading() - """ - if file_handle is not None: - file_handle.close() - def reader_finished(self, reader): self.readers -= 1 return defer.succeed(True) diff --git a/lbrynet/blob/reader.py b/lbrynet/blob/reader.py index 745e62ef8..afd62e57e 100644 --- a/lbrynet/blob/reader.py +++ b/lbrynet/blob/reader.py @@ -1,39 +1,8 @@ import logging -from twisted.internet import interfaces -from zope.interface import implements log = logging.getLogger(__name__) -class HashBlobReader_v0(object): - """ - This is a class that is only used in StreamBlobDecryptor - and should be deprecated - """ - implements(interfaces.IConsumer) - - def __init__(self, write_func): - self.write_func = write_func - - def registerProducer(self, producer, streaming): - from twisted.internet import reactor - - self.producer = producer - self.streaming = streaming - if self.streaming is False: - reactor.callLater(0, self.producer.resumeProducing) - - def unregisterProducer(self): - pass - - def write(self, data): - from twisted.internet import reactor - - self.write_func(data) - if self.streaming is False: - reactor.callLater(0, self.producer.resumeProducing) - - class HashBlobReader(object): """ This is a file like reader class that supports From 51d466876300a4e3dea366ee1749081fda0b3d43 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Fri, 29 Sep 2017 14:43:29 -0400 Subject: [PATCH 13/17] fix test --- lbrynet/tests/unit/cryptstream/test_cryptblob.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lbrynet/tests/unit/cryptstream/test_cryptblob.py b/lbrynet/tests/unit/cryptstream/test_cryptblob.py index 083d2e1fc..1631b68b0 100644 --- a/lbrynet/tests/unit/cryptstream/test_cryptblob.py +++ b/lbrynet/tests/unit/cryptstream/test_cryptblob.py @@ -9,6 +9,7 @@ from Crypto import Random from Crypto.Cipher import AES import random import string +import StringIO class MocBlob(object): def __init__(self): @@ -19,6 +20,9 @@ class MocBlob(object): write_func(data) return defer.succeed(True) + def open_for_reading(self): + return StringIO.StringIO(self.data) + def write(self, data): self.data += data @@ -64,7 +68,7 @@ class TestCryptBlob(unittest.TestCase): # decrypt string decryptor = CryptBlob.StreamBlobDecryptor(blob, key, iv, size_of_data) - decryptor.decrypt(write_func) + yield decryptor.decrypt(write_func) self.assertEqual(self.data_buf, string_to_encrypt) @defer.inlineCallbacks From 9470b318b0021df41a14494f23922d47b0ab46ae Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 25 Sep 2017 12:12:40 -0400 Subject: [PATCH 14/17] moved BLOB_SIZE in conf to MAX_BLOB_SIZE constant in blob.blob_file --- lbrynet/blob/blob_file.py | 5 ++--- lbrynet/conf.py | 1 - lbrynet/core/client/ClientRequest.py | 5 ++--- lbrynet/cryptstream/CryptBlob.py | 5 ++--- lbrynet/tests/unit/cryptstream/test_cryptblob.py | 4 ++-- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lbrynet/blob/blob_file.py b/lbrynet/blob/blob_file.py index 5c08b2059..42b402030 100644 --- a/lbrynet/blob/blob_file.py +++ b/lbrynet/blob/blob_file.py @@ -3,15 +3,14 @@ import os from twisted.internet import defer, threads from twisted.web.client import FileBodyProducer from twisted.python.failure import Failure -from lbrynet import conf from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError from lbrynet.core.utils import is_valid_blobhash from lbrynet.blob.writer import HashBlobWriter from lbrynet.blob.reader import HashBlobReader - log = logging.getLogger(__name__) +MAX_BLOB_SIZE = 2 * 2 ** 20 class BlobFile(object): """ @@ -124,7 +123,7 @@ class BlobFile(object): def set_length(self, length): if self.length is not None and length == self.length: return True - if self.length is None and 0 <= length <= conf.settings['BLOB_SIZE']: + if self.length is None and 0 <= length <= MAX_BLOB_SIZE: self.length = length return True log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 4b156ad89..cc290a3bf 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -206,7 +206,6 @@ FIXED_SETTINGS = { 'API_ADDRESS': 'lbryapi', 'APP_NAME': APP_NAME, 'BLOBFILES_DIR': 'blobfiles', - 'BLOB_SIZE': 2 * MB, 'CRYPTSD_FILE_EXTENSION': '.cryptsd', 'CURRENCIES': { 'BTC': {'type': 'crypto'}, diff --git a/lbrynet/core/client/ClientRequest.py b/lbrynet/core/client/ClientRequest.py index 04c3dac7f..1dee9b9d6 100644 --- a/lbrynet/core/client/ClientRequest.py +++ b/lbrynet/core/client/ClientRequest.py @@ -1,5 +1,4 @@ -from lbrynet import conf - +from lbrynet.blob.blob_file import MAX_BLOB_SIZE class ClientRequest(object): def __init__(self, request_dict, response_identifier=None): @@ -17,7 +16,7 @@ class ClientBlobRequest(ClientPaidRequest): def __init__(self, request_dict, response_identifier, write_func, finished_deferred, cancel_func, blob): if blob.length is None: - max_pay_units = conf.settings['BLOB_SIZE'] + max_pay_units = MAX_BLOB_SIZE else: max_pay_units = blob.length ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units) diff --git a/lbrynet/cryptstream/CryptBlob.py b/lbrynet/cryptstream/CryptBlob.py index 6d4bcbc5e..08c0fe09e 100644 --- a/lbrynet/cryptstream/CryptBlob.py +++ b/lbrynet/cryptstream/CryptBlob.py @@ -5,9 +5,8 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes from cryptography.hazmat.primitives.ciphers.algorithms import AES from cryptography.hazmat.primitives.padding import PKCS7 from cryptography.hazmat.backends import default_backend -from lbrynet import conf from lbrynet.core.BlobInfo import BlobInfo - +from lbrynet.blob.blob_file import MAX_BLOB_SIZE log = logging.getLogger(__name__) backend = default_backend() @@ -114,7 +113,7 @@ class CryptStreamBlobMaker(object): max bytes are written. num_bytes_to_write is the number of bytes that will be written from data in this call """ - max_bytes_to_write = conf.settings['BLOB_SIZE'] - self.length - 1 + max_bytes_to_write = MAX_BLOB_SIZE - self.length - 1 done = False if max_bytes_to_write <= len(data): num_bytes_to_write = max_bytes_to_write diff --git a/lbrynet/tests/unit/cryptstream/test_cryptblob.py b/lbrynet/tests/unit/cryptstream/test_cryptblob.py index 1631b68b0..2378c5770 100644 --- a/lbrynet/tests/unit/cryptstream/test_cryptblob.py +++ b/lbrynet/tests/unit/cryptstream/test_cryptblob.py @@ -1,7 +1,7 @@ from twisted.trial import unittest from twisted.internet import defer from lbrynet.cryptstream import CryptBlob -from lbrynet import conf +from lbrynet.blob.blob_file import MAX_BLOB_SIZE from lbrynet.tests.mocks import mock_conf_settings @@ -57,7 +57,7 @@ class TestCryptBlob(unittest.TestCase): expected_encrypted_blob_size = ((size_of_data / AES.block_size) + 1) * AES.block_size self.assertEqual(expected_encrypted_blob_size, len(blob.data)) - if size_of_data < conf.settings['BLOB_SIZE']-1: + if size_of_data < MAX_BLOB_SIZE-1: self.assertFalse(done) else: self.assertTrue(done) From a5293de44b837ce5823e50a5bf2f57b3c65f3dfd Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Fri, 29 Sep 2017 15:23:49 -0400 Subject: [PATCH 15/17] adding changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 820e64931..59498f825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ at anytime. * ### Changed - * + * Moved BLOB_SIZE from conf.py to MAX_BLOB_SIZE in blob/blob_file.py * ### Added @@ -30,7 +30,7 @@ at anytime. * ### Removed - * + * Removed some alternate methods of reading from blob files * From 10ac86a99e052c18f6be80b6d4f678c0ed2c65a7 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Wed, 25 Oct 2017 16:04:35 -0400 Subject: [PATCH 16/17] use FileBodyProducer to read --- lbrynet/cryptstream/CryptBlob.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lbrynet/cryptstream/CryptBlob.py b/lbrynet/cryptstream/CryptBlob.py index 08c0fe09e..c99465673 100644 --- a/lbrynet/cryptstream/CryptBlob.py +++ b/lbrynet/cryptstream/CryptBlob.py @@ -1,6 +1,8 @@ import binascii import logging -from twisted.internet import defer, threads +from io import BytesIO +from twisted.internet import defer +from twisted.web.client import FileBodyProducer from cryptography.hazmat.primitives.ciphers import Cipher, modes from cryptography.hazmat.primitives.ciphers.algorithms import AES from cryptography.hazmat.primitives.padding import PKCS7 @@ -73,14 +75,17 @@ class StreamBlobDecryptor(object): read_handle = self.blob.open_for_reading() + @defer.inlineCallbacks def decrypt_bytes(): - data = read_handle.read() - self.buff += data - self.len_read += len(data) + producer = FileBodyProducer(read_handle) + buff = BytesIO() + yield producer.startProducing(buff) + self.buff = buff.getvalue() + self.len_read += len(self.buff) write_bytes() finish_decrypt() - d = threads.deferToThread(decrypt_bytes) + d = decrypt_bytes() return d From c86db2d84627df0f09f699c65595c522c36ced7b Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 1 Nov 2017 17:28:07 -0400 Subject: [PATCH 17/17] Bump version 0.17.2rc2 --> 0.17.2rc3 Signed-off-by: Jack Robison --- lbrynet/__init__.py | 2 +- requirements.txt | 4 ++-- setup.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 4073da61a..d22bf7b56 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,6 +1,6 @@ import logging -__version__ = "0.17.2rc2" +__version__ = "0.17.2rc3" version = tuple(__version__.split('.')) logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/requirements.txt b/requirements.txt index b060e18f0..801772549 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,8 @@ gmpy==1.17 jsonrpc==1.2 jsonrpclib==0.1.7 jsonschema==2.6.0 -git+https://github.com/lbryio/lbryum.git@v3.1.10#egg=lbryum -git+https://github.com/lbryio/lbryschema.git@v0.0.13#egg=lbryschema +git+https://github.com/lbryio/lbryum.git@v3.1.11rc1#egg=lbryum +git+https://github.com/lbryio/lbryschema.git@v0.0.14rc1#egg=lbryschema miniupnpc==1.9 pbkdf2==1.3 pycrypto==2.6.1 diff --git a/setup.py b/setup.py index 21f5b018d..6d6b01ffe 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ requires = [ 'envparse', 'jsonrpc', 'jsonschema', - 'lbryum==3.1.10', - 'lbryschema==0.0.13', + 'lbryum==3.1.11rc1', + 'lbryschema==0.0.14rc1', 'miniupnpc', 'pycrypto', 'pyyaml',