From a38e5c855e5e23a26c4297c0c3d8f5ebe0cb2346 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Mon, 19 Dec 2016 18:16:37 -0800 Subject: [PATCH 1/8] Remove upload_allowed attribute This is unused and largely pulluting the argument list of too many functions. --- lbrynet/core/BlobManager.py | 26 +++++++------- lbrynet/core/HashBlob.py | 3 +- lbrynet/core/client/DownloadManager.py | 6 ++-- .../core/client/StandaloneBlobDownloader.py | 2 +- .../client/CryptStreamDownloader.py | 5 ++- .../client/EncryptedFileDownloader.py | 34 ++++++++----------- .../EncryptedFileDownloader.py | 8 ++--- .../lbryfilemanager/EncryptedFileManager.py | 7 ++-- .../lbrylive/client/LiveStreamDownloader.py | 13 ++++--- lbrynet/lbrynet_daemon/Daemon.py | 3 -- lbrynet/reflector/server/server.py | 4 +-- 11 files changed, 47 insertions(+), 64 deletions(-) diff --git a/lbrynet/core/BlobManager.py b/lbrynet/core/BlobManager.py index 97faf1d91..a3d977767 100644 --- a/lbrynet/core/BlobManager.py +++ b/lbrynet/core/BlobManager.py @@ -24,13 +24,13 @@ class BlobManager(DHTHashSupplier): def setup(self): pass - def get_blob(self, blob_hash, upload_allowed, length): + def get_blob(self, blob_hash, length=None): pass def get_blob_creator(self): pass - def _make_new_blob(self, blob_hash, upload_allowed, length): + def _make_new_blob(self, blob_hash, length): pass def blob_completed(self, blob, next_announce_time=None): @@ -107,22 +107,20 @@ class DiskBlobManager(BlobManager): self.db_conn = None return defer.succeed(True) - def get_blob(self, blob_hash, upload_allowed, length=None): + def get_blob(self, blob_hash, length=None): """Return a blob identified by blob_hash, which may be a new blob or a blob that is already on the hard disk """ - # TODO: if blob.upload_allowed and upload_allowed is False, - # change upload_allowed in blob and on disk if blob_hash in self.blobs: return defer.succeed(self.blobs[blob_hash]) - return self._make_new_blob(blob_hash, upload_allowed, length) + return self._make_new_blob(blob_hash, length) def get_blob_creator(self): return self.blob_creator_type(self, self.blob_dir) - def _make_new_blob(self, blob_hash, upload_allowed, length=None): + def _make_new_blob(self, blob_hash, length=None): log.debug('Making a new blob for %s', blob_hash) - blob = self.blob_type(self.blob_dir, blob_hash, upload_allowed, length) + blob = self.blob_type(self.blob_dir, blob_hash, length) self.blobs[blob_hash] = blob return defer.succeed(blob) @@ -144,7 +142,7 @@ class DiskBlobManager(BlobManager): assert blob_creator.blob_hash is not None assert blob_creator.blob_hash not in self.blobs assert blob_creator.length is not None - new_blob = self.blob_type(self.blob_dir, blob_creator.blob_hash, True, blob_creator.length) + new_blob = self.blob_type(self.blob_dir, blob_creator.blob_hash, blob_creator.length) self.blobs[blob_creator.blob_hash] = new_blob self._immediate_announce([blob_creator.blob_hash]) next_announce_time = self.get_next_announce_time() @@ -364,16 +362,16 @@ class TempBlobManager(BlobManager): self._next_manage_call.cancel() self._next_manage_call = None - def get_blob(self, blob_hash, upload_allowed, length=None): + def get_blob(self, blob_hash, length=None): if blob_hash in self.blobs: return defer.succeed(self.blobs[blob_hash]) - return self._make_new_blob(blob_hash, upload_allowed, length) + return self._make_new_blob(blob_hash, length) def get_blob_creator(self): return self.blob_creator_type(self) - def _make_new_blob(self, blob_hash, upload_allowed, length=None): - blob = self.blob_type(blob_hash, upload_allowed, length) + def _make_new_blob(self, blob_hash, length=None): + blob = self.blob_type(blob_hash, length) self.blobs[blob_hash] = blob return defer.succeed(blob) @@ -409,7 +407,7 @@ class TempBlobManager(BlobManager): assert blob_creator.blob_hash is not None assert blob_creator.blob_hash not in self.blobs assert blob_creator.length is not None - new_blob = self.blob_type(blob_creator.blob_hash, True, blob_creator.length) + new_blob = self.blob_type(blob_creator.blob_hash, blob_creator.length) # TODO: change this; its breaks the encapsulation of the # blob. Maybe better would be to have the blob_creator # produce a blob. diff --git a/lbrynet/core/HashBlob.py b/lbrynet/core/HashBlob.py index 929d5637e..97ce31218 100644 --- a/lbrynet/core/HashBlob.py +++ b/lbrynet/core/HashBlob.py @@ -82,14 +82,13 @@ class HashBlobWriter(object): class HashBlob(object): """A chunk of data available on the network which is specified by a hashsum""" - def __init__(self, blob_hash, upload_allowed, length=None): + def __init__(self, blob_hash, length=None): assert is_valid_blobhash(blob_hash) self.blob_hash = blob_hash self.length = length self.writers = {} # {Peer: writer, finished_deferred} self.finished_deferred = None self._verified = False - self.upload_allowed = upload_allowed self.readers = 0 @property diff --git a/lbrynet/core/client/DownloadManager.py b/lbrynet/core/client/DownloadManager.py index dedd2e3df..a802d7841 100644 --- a/lbrynet/core/client/DownloadManager.py +++ b/lbrynet/core/client/DownloadManager.py @@ -11,9 +11,8 @@ log = logging.getLogger(__name__) class DownloadManager(object): implements(interfaces.IDownloadManager) - def __init__(self, blob_manager, upload_allowed): + def __init__(self, blob_manager): self.blob_manager = blob_manager - self.upload_allowed = upload_allowed self.blob_info_finder = None self.progress_manager = None self.blob_handler = None @@ -82,8 +81,7 @@ class DownloadManager(object): self.blob_infos[blob_info.blob_num] = blob_info log.debug( "Trying to get the blob associated with blob hash %s", blob_info.blob_hash) - d = self.blob_manager.get_blob( - blob_info.blob_hash, self.upload_allowed, blob_info.length) + d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length) d.addCallback(add_blob_to_list, blob_info.blob_num) d.addErrback(error_during_add) ds.append(d) diff --git a/lbrynet/core/client/StandaloneBlobDownloader.py b/lbrynet/core/client/StandaloneBlobDownloader.py index aa5204d8d..e4f44ca6e 100644 --- a/lbrynet/core/client/StandaloneBlobDownloader.py +++ b/lbrynet/core/client/StandaloneBlobDownloader.py @@ -112,7 +112,7 @@ class StandaloneBlobDownloader(object): self.stop() self.finished_deferred = defer.Deferred(canceller=cancel_download) - self.download_manager = DownloadManager(self.blob_manager, True) + self.download_manager = DownloadManager(self.blob_manager) self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder, self.payment_rate_manager, self.wallet, self.download_manager) diff --git a/lbrynet/cryptstream/client/CryptStreamDownloader.py b/lbrynet/cryptstream/client/CryptStreamDownloader.py index 657b8ca5e..9d2ba3b4f 100644 --- a/lbrynet/cryptstream/client/CryptStreamDownloader.py +++ b/lbrynet/cryptstream/client/CryptStreamDownloader.py @@ -38,7 +38,7 @@ class CryptStreamDownloader(object): implements(IStreamDownloader) def __init__(self, peer_finder, rate_limiter, blob_manager, - payment_rate_manager, wallet, upload_allowed): + payment_rate_manager, wallet): """Initialize a CryptStreamDownloader @param peer_finder: An object which implements the IPeerFinder @@ -61,7 +61,6 @@ class CryptStreamDownloader(object): self.blob_manager = blob_manager self.payment_rate_manager = payment_rate_manager self.wallet = wallet - self.upload_allowed = upload_allowed self.key = None self.stream_name = None self.completed = False @@ -146,7 +145,7 @@ class CryptStreamDownloader(object): def _get_download_manager(self): assert self.blob_requester is None - download_manager = DownloadManager(self.blob_manager, self.upload_allowed) + download_manager = DownloadManager(self.blob_manager) # TODO: can we get rid of these circular references. I'm not # smart enough to handle thinking about the interactions # between them and have hope that there is a simpler way diff --git a/lbrynet/lbryfile/client/EncryptedFileDownloader.py b/lbrynet/lbryfile/client/EncryptedFileDownloader.py index 6864ecf8e..f8e4396de 100644 --- a/lbrynet/lbryfile/client/EncryptedFileDownloader.py +++ b/lbrynet/lbryfile/client/EncryptedFileDownloader.py @@ -23,9 +23,9 @@ class EncryptedFileDownloader(CryptStreamDownloader): """Classes which inherit from this class download LBRY files""" def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, - stream_info_manager, payment_rate_manager, wallet, upload_allowed): + stream_info_manager, payment_rate_manager, wallet): CryptStreamDownloader.__init__(self, peer_finder, rate_limiter, blob_manager, - payment_rate_manager, wallet, upload_allowed) + payment_rate_manager, wallet) self.stream_hash = stream_hash self.stream_info_manager = stream_info_manager self.suggested_file_name = None @@ -141,7 +141,6 @@ class EncryptedFileDownloaderFactory(object): def make_downloader(self, metadata, options, payment_rate_manager, **kwargs): payment_rate_manager.min_blob_data_payment_rate = options[0] - upload_allowed = options[1] def save_source_if_blob(stream_hash): if metadata.metadata_source == StreamMetadata.FROM_BLOB: @@ -154,7 +153,7 @@ class EncryptedFileDownloaderFactory(object): def create_downloader(stream_hash): downloader = self._make_downloader(stream_hash, payment_rate_manager, - metadata.validator.raw_info, upload_allowed) + metadata.validator.raw_info) d = downloader.set_stream_info() d.addCallback(lambda _: downloader) return d @@ -164,18 +163,17 @@ class EncryptedFileDownloaderFactory(object): d.addCallback(create_downloader) return d - def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed): + def _make_downloader(self, stream_hash, payment_rate_manager, stream_info): pass class EncryptedFileSaver(EncryptedFileDownloader): def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, - payment_rate_manager, wallet, download_directory, upload_allowed, file_name=None): + payment_rate_manager, wallet, download_directory, file_name=None): EncryptedFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, - payment_rate_manager, wallet, - upload_allowed) + payment_rate_manager, wallet) self.download_directory = download_directory self.file_name = file_name self.file_written_to = None @@ -208,9 +206,7 @@ class EncryptedFileSaver(EncryptedFileDownloader): def _get_progress_manager(self, download_manager): return FullStreamProgressManager(self._finished_downloading, self.blob_manager, - download_manager, - delete_blob_after_finished=not - self.upload_allowed) + download_manager) def _setup_output(self): def open_file(): @@ -274,12 +270,12 @@ class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory): stream_info_manager, wallet) self.download_directory = download_directory - def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed): + def _make_downloader(self, stream_hash, payment_rate_manager, stream_info): return EncryptedFileSaver(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.stream_info_manager, payment_rate_manager, self.wallet, - self.download_directory, upload_allowed) + self.download_directory) @staticmethod def get_description(): @@ -288,12 +284,12 @@ class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory): class EncryptedFileOpener(EncryptedFileDownloader): def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, - payment_rate_manager, wallet, upload_allowed): + payment_rate_manager, wallet): EncryptedFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, payment_rate_manager, wallet, - upload_allowed) + ) self.process = None self.process_log = None @@ -305,9 +301,7 @@ class EncryptedFileOpener(EncryptedFileDownloader): def _get_progress_manager(self, download_manager): return FullStreamProgressManager(self._finished_downloading, self.blob_manager, - download_manager, - delete_blob_after_finished=not - self.upload_allowed) + download_manager) def _setup_output(self): def start_process(): @@ -363,12 +357,12 @@ class EncryptedFileOpenerFactory(EncryptedFileDownloaderFactory): return True return False - def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed): + def _make_downloader(self, stream_hash, payment_rate_manager, stream_info): return EncryptedFileOpener(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.stream_info_manager, payment_rate_manager, self.wallet, - upload_allowed) + ) @staticmethod def get_description(): diff --git a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py index e1035df31..14f127ecd 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py +++ b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py @@ -25,13 +25,13 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, lbry_file_manager, payment_rate_manager, wallet, download_directory, - upload_allowed, file_name=None): + file_name=None): EncryptedFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, payment_rate_manager, wallet, download_directory, - upload_allowed, file_name) + file_name) self.sd_hash = None self.txid = None self.nout = None @@ -151,15 +151,15 @@ class ManagedEncryptedFileDownloaderFactory(object): @defer.inlineCallbacks def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, file_name=None): + # TODO: change options argument to only be data_rate data_rate = options[0] - upload_allowed = options[1] stream_hash = yield save_sd_info(self.lbry_file_manager.stream_info_manager, metadata.validator.raw_info) if metadata.metadata_source == StreamMetadata.FROM_BLOB: yield self.lbry_file_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash) lbry_file = yield self.lbry_file_manager.add_lbry_file(stream_hash, payment_rate_manager, - data_rate, upload_allowed, + data_rate, download_directory, file_name) defer.returnValue(lbry_file) diff --git a/lbrynet/lbryfilemanager/EncryptedFileManager.py b/lbrynet/lbryfilemanager/EncryptedFileManager.py index b4ad6b7c4..d00862b10 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileManager.py +++ b/lbrynet/lbryfilemanager/EncryptedFileManager.py @@ -137,7 +137,7 @@ class EncryptedFileManager(object): @defer.inlineCallbacks def start_lbry_file(self, rowid, stream_hash, - payment_rate_manager, blob_data_rate=None, upload_allowed=True, + payment_rate_manager, blob_data_rate=None, download_directory=None, file_name=None): if not download_directory: download_directory = self.download_directory @@ -153,7 +153,6 @@ class EncryptedFileManager(object): payment_rate_manager, self.session.wallet, download_directory, - upload_allowed, file_name=file_name ) yield lbry_file_downloader.set_stream_info() @@ -183,10 +182,10 @@ class EncryptedFileManager(object): @defer.inlineCallbacks def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, - upload_allowed=True, download_directory=None, file_name=None): + download_directory=None, file_name=None): rowid = yield self._save_lbry_file(stream_hash, blob_data_rate) lbry_file = yield self.start_lbry_file(rowid, stream_hash, payment_rate_manager, - blob_data_rate, upload_allowed, download_directory, + blob_data_rate, download_directory, file_name) defer.returnValue(lbry_file) diff --git a/lbrynet/lbrylive/client/LiveStreamDownloader.py b/lbrynet/lbrylive/client/LiveStreamDownloader.py index 3766ec579..913aba7dc 100644 --- a/lbrynet/lbrylive/client/LiveStreamDownloader.py +++ b/lbrynet/lbrylive/client/LiveStreamDownloader.py @@ -16,9 +16,9 @@ from lbrynet.lbrylive.StreamDescriptor import LiveStreamType class _LiveStreamDownloader(CryptStreamDownloader): def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, - payment_rate_manager, wallet, upload_allowed): + payment_rate_manager, wallet): CryptStreamDownloader.__init__(self, peer_finder, rate_limiter, blob_manager, - payment_rate_manager, wallet, upload_allowed) + payment_rate_manager, wallet) self.stream_hash = stream_hash self.stream_info_manager = stream_info_manager self.public_key = None @@ -42,9 +42,9 @@ class _LiveStreamDownloader(CryptStreamDownloader): class LiveStreamDownloader(_LiveStreamDownloader): def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, - payment_rate_manager, wallet, upload_allowed): + payment_rate_manager, wallet): _LiveStreamDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, - stream_info_manager, payment_rate_manager, wallet, upload_allowed) + stream_info_manager, payment_rate_manager, wallet) def _get_metadata_handler(self, download_manager): @@ -66,10 +66,10 @@ class LiveStreamDownloader(_LiveStreamDownloader): class FullLiveStreamDownloader(_LiveStreamDownloader): def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, - payment_rate_manager, wallet, upload_allowed): + payment_rate_manager, wallet): _LiveStreamDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, payment_rate_manager, - wallet, upload_allowed) + wallet) self.file_handle = None self.file_name = None @@ -161,7 +161,6 @@ class FullLiveStreamDownloaderFactory(object): stream_downloader = FullLiveStreamDownloader(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.stream_info_manager, prm, self.wallet, True) - # TODO: change upload_allowed=True above to something better d = stream_downloader.set_stream_info() d.addCallback(lambda _: stream_downloader) return d diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index fc1a34e58..fdc730a2b 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -1357,7 +1357,6 @@ class Daemon(AuthJSONRPCServer): 'stream_hash': base 58 string 'stream_name': string 'suggested_file_name': string - 'upload_allowed': bool 'sd_hash': string """ @@ -1389,7 +1388,6 @@ class Daemon(AuthJSONRPCServer): 'stream_hash': base 58 string 'stream_name': string 'suggested_file_name': string - 'upload_allowed': bool 'sd_hash': string """ d = self._get_deferred_for_lbry_file(kwargs) @@ -2583,7 +2581,6 @@ class _GetFileHelper(object): 'stream_hash': lbry_file.stream_hash, 'stream_name': lbry_file.stream_name, 'suggested_file_name': lbry_file.suggested_file_name, - 'upload_allowed': lbry_file.upload_allowed, 'sd_hash': lbry_file.sd_hash, 'lbry_uri': lbry_file.uri, 'txid': lbry_file.txid, diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index 5340ba248..e3dd8685f 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -295,10 +295,10 @@ class ReflectorServer(Protocol): if self.blob_write is None: log.debug('Received info for blob: %s', blob_hash[:16]) - d = self.blob_manager.get_blob(blob_hash, True, blob_size) + d = self.blob_manager.get_blob(blob_hash, blob_size) d.addCallback(self.get_blob_response) d.addCallback(self.send_response) - else: + else: log.debug('blob is already open') self.receiving_blob = True d = self.blob_finished_d From 439a840700f7f17904c92c40eeb34cfa2a878827 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Mon, 19 Dec 2016 18:17:48 -0800 Subject: [PATCH 2/8] Better log in tests on failure --- tests/functional/test_misc.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index a9ef36d08..1b028ae38 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -1,3 +1,5 @@ +from lbrynet.core import log_support + import io import logging from multiprocessing import Process, Event, Queue @@ -59,6 +61,7 @@ GenFile = mocks.GenFile test_create_stream_sd_file = mocks.create_stream_sd_file DummyBlobAvailabilityTracker = mocks.BlobAvailabilityTracker +log = logging.getLogger(__name__) log_format = "%(funcName)s(): %(message)s" logging.basicConfig(level=logging.WARNING, format=log_format) @@ -139,11 +142,7 @@ class LbryUploader(object): d.addCallback(lambda _: self.create_stream()) d.addCallback(self.create_stream_descriptor) d.addCallback(self.put_sd_hash_on_queue) - - def print_error(err): - logging.critical("Server error: %s", err.getErrorMessage()) - - d.addErrback(print_error) + d.addErrback(log.fail(), 'Failed to start') return d def start_server(self): @@ -473,11 +472,7 @@ def start_blob_uploader(blob_hash_queue, kill_event, dead_event, slow, is_genero d.addCallback(lambda _: start_server()) d.addCallback(lambda _: create_single_blob()) d.addCallback(put_blob_hash_on_queue) - - def print_error(err): - logging.critical("Server error: %s", err.getErrorMessage()) - - d.addErrback(print_error) + d.addErrback(log.fail(), 'Failed to start') return d def start_server(): From 2a45c911756d3a54a4e5f1a010bdadc1dc09850d Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 18 Jan 2017 21:08:02 -0600 Subject: [PATCH 3/8] fix get_blob calls with upload_allowed arguments --- lbrynet/core/server/BlobRequestHandler.py | 4 ++-- lbrynet/lbrynet_daemon/Daemon.py | 2 +- lbrynet/reflector/client/blob.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lbrynet/core/server/BlobRequestHandler.py b/lbrynet/core/server/BlobRequestHandler.py index aea8cd276..2d08f25c2 100644 --- a/lbrynet/core/server/BlobRequestHandler.py +++ b/lbrynet/core/server/BlobRequestHandler.py @@ -138,7 +138,7 @@ class BlobRequestHandler(object): return self._send_blob(response, query) def _send_blob(self, response, query): - d = self.blob_manager.get_blob(query, True) + d = self.blob_manager.get_blob(query) d.addCallback(self.open_blob_for_reading, response) return d @@ -177,7 +177,7 @@ class BlobRequestHandler(object): return defer.succeed(response) else: log.debug("Requested blob: %s", str(incoming)) - d = self.blob_manager.get_blob(incoming, True) + d = self.blob_manager.get_blob(incoming) d.addCallback(lambda blob: self.open_blob_for_reading(blob, response)) return d diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index fdc730a2b..d3224cda4 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -858,7 +858,7 @@ class Daemon(AuthJSONRPCServer): def _get_or_download_sd_blob(self, blob, sd_hash): if blob: - return self.session.blob_manager.get_blob(blob[0], True) + return self.session.blob_manager.get_blob(blob[0]) def _check_est(downloader): if downloader.result is not None: diff --git a/lbrynet/reflector/client/blob.py b/lbrynet/reflector/client/blob.py index 2aba7716f..854dc6489 100644 --- a/lbrynet/reflector/client/blob.py +++ b/lbrynet/reflector/client/blob.py @@ -164,7 +164,7 @@ class BlobReflectorClient(Protocol): blob_hash = self.blob_hashes_to_send[0] log.debug('No current blob, sending the next one: %s', blob_hash) self.blob_hashes_to_send = self.blob_hashes_to_send[1:] - d = self.blob_manager.get_blob(blob_hash, True) + d = self.blob_manager.get_blob(blob_hash) d.addCallback(self.open_blob_for_reading) # send the server the next blob hash + length d.addCallbacks( From fe4dac8236f7a83d29d7469155c23e38aeb91aab Mon Sep 17 00:00:00 2001 From: jobevers Date: Tue, 14 Feb 2017 11:32:04 -0600 Subject: [PATCH 4/8] change downloader options to data_rate --- lbrynet/lbryfile/client/EncryptedFileDownloader.py | 4 ++-- lbrynet/lbryfilemanager/EncryptedFileDownloader.py | 4 +--- lbrynet/lbrylive/client/LiveStreamDownloader.py | 2 +- lbrynet/lbrynet_daemon/Downloader.py | 12 +++++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lbrynet/lbryfile/client/EncryptedFileDownloader.py b/lbrynet/lbryfile/client/EncryptedFileDownloader.py index f8e4396de..092c5ae8a 100644 --- a/lbrynet/lbryfile/client/EncryptedFileDownloader.py +++ b/lbrynet/lbryfile/client/EncryptedFileDownloader.py @@ -139,8 +139,8 @@ class EncryptedFileDownloaderFactory(object): def can_download(self, sd_validator): return True - def make_downloader(self, metadata, options, payment_rate_manager, **kwargs): - payment_rate_manager.min_blob_data_payment_rate = options[0] + def make_downloader(self, metadata, data_rate, payment_rate_manager, **kwargs): + payment_rate_manager.min_blob_data_payment_rate = data_rate def save_source_if_blob(stream_hash): if metadata.metadata_source == StreamMetadata.FROM_BLOB: diff --git a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py index 14f127ecd..b257f4994 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py +++ b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py @@ -149,10 +149,8 @@ class ManagedEncryptedFileDownloaderFactory(object): return True @defer.inlineCallbacks - def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, + def make_downloader(self, metadata, data_rate, payment_rate_manager, download_directory=None, file_name=None): - # TODO: change options argument to only be data_rate - data_rate = options[0] stream_hash = yield save_sd_info(self.lbry_file_manager.stream_info_manager, metadata.validator.raw_info) if metadata.metadata_source == StreamMetadata.FROM_BLOB: diff --git a/lbrynet/lbrylive/client/LiveStreamDownloader.py b/lbrynet/lbrylive/client/LiveStreamDownloader.py index 913aba7dc..4b152f22a 100644 --- a/lbrynet/lbrylive/client/LiveStreamDownloader.py +++ b/lbrynet/lbrylive/client/LiveStreamDownloader.py @@ -141,7 +141,7 @@ class FullLiveStreamDownloaderFactory(object): def can_download(self, sd_validator): return True - def make_downloader(self, metadata, options, payment_rate_manager): + def make_downloader(self, metadata, data_rate, payment_rate_manager): # TODO: check options for payment rate manager parameters prm = LiveStreamPaymentRateManager(self.default_payment_rate_manager, payment_rate_manager) diff --git a/lbrynet/lbrynet_daemon/Downloader.py b/lbrynet/lbrynet_daemon/Downloader.py index abcc351ac..0db02ee44 100644 --- a/lbrynet/lbrynet_daemon/Downloader.py +++ b/lbrynet/lbrynet_daemon/Downloader.py @@ -119,11 +119,13 @@ class GetStream(object): @defer.inlineCallbacks def get_downloader(self, factory, stream_metadata): - downloader_options = [self.data_rate, True] - downloader = yield factory.make_downloader(stream_metadata, downloader_options, - self.payment_rate_manager, - download_directory=self.download_directory, - file_name=self.file_name) + downloader = yield factory.make_downloader( + stream_metadata, + self.data_rate, + self.payment_rate_manager, + download_directory=self.download_directory, + file_name=self.file_name + ) defer.returnValue(downloader) def _pay_key_fee(self, address, fee_lbc, name): From 4fb5e932e6aed8dd805171ce0c33b842206a666c Mon Sep 17 00:00:00 2001 From: jobevers Date: Tue, 14 Feb 2017 11:35:06 -0600 Subject: [PATCH 5/8] fix indentation --- lbrynet/reflector/server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index e3dd8685f..83f04f864 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -298,7 +298,7 @@ class ReflectorServer(Protocol): d = self.blob_manager.get_blob(blob_hash, blob_size) d.addCallback(self.get_blob_response) d.addCallback(self.send_response) - else: + else: log.debug('blob is already open') self.receiving_blob = True d = self.blob_finished_d From 9e2cda0cdcfd05af1a1105af86b6bec1e16e097c Mon Sep 17 00:00:00 2001 From: jobevers Date: Tue, 14 Feb 2017 13:18:42 -0600 Subject: [PATCH 6/8] remove more upload_allowed arguments --- lbrynet/core/BlobManager.py | 5 +++-- lbrynet/lbryfile/client/EncryptedFileDownloader.py | 4 +++- lbrynet/lbryfile/client/EncryptedFileOptions.py | 12 ------------ lbrynet/lbryfilemanager/EncryptedFileDownloader.py | 4 +++- lbrynet/lbryfilemanager/EncryptedFileManager.py | 6 +++--- lbrynet/lbrylive/client/LiveStreamDownloader.py | 2 +- lbrynet/lbrynet_daemon/Daemon.py | 2 +- lbrynet/lbrynet_daemon/Downloader.py | 4 +++- lbrynet/reflector/client/client.py | 4 ++-- lbrynet/reflector/server/server.py | 4 ++-- tests/functional/test_misc.py | 2 +- tests/functional/test_reflector.py | 6 +++--- 12 files changed, 25 insertions(+), 30 deletions(-) diff --git a/lbrynet/core/BlobManager.py b/lbrynet/core/BlobManager.py index a3d977767..5a2afb713 100644 --- a/lbrynet/core/BlobManager.py +++ b/lbrynet/core/BlobManager.py @@ -111,6 +111,7 @@ class DiskBlobManager(BlobManager): """Return a blob identified by blob_hash, which may be a new blob or a blob that is already on the hard disk """ + assert length is None or isinstance(length, int) if blob_hash in self.blobs: return defer.succeed(self.blobs[blob_hash]) return self._make_new_blob(blob_hash, length) @@ -216,7 +217,7 @@ class DiskBlobManager(BlobManager): for blob_hash, being_deleted in self.blob_hashes_to_delete.items(): if being_deleted is False: self.blob_hashes_to_delete[blob_hash] = True - d = self.get_blob(blob_hash, True) + d = self.get_blob(blob_hash) d.addCallbacks( delete, set_not_deleting, callbackArgs=(blob_hash,), errbackArgs=(blob_hash,)) @@ -270,7 +271,7 @@ class DiskBlobManager(BlobManager): @defer.inlineCallbacks def _completed_blobs(self, blobhashes_to_check): """Returns of the blobhashes_to_check, which are valid""" - blobs = yield defer.DeferredList([self.get_blob(b, True) for b in blobhashes_to_check]) + blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check]) blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified] defer.returnValue(blob_hashes) diff --git a/lbrynet/lbryfile/client/EncryptedFileDownloader.py b/lbrynet/lbryfile/client/EncryptedFileDownloader.py index 092c5ae8a..fba1c0461 100644 --- a/lbrynet/lbryfile/client/EncryptedFileDownloader.py +++ b/lbrynet/lbryfile/client/EncryptedFileDownloader.py @@ -139,7 +139,9 @@ class EncryptedFileDownloaderFactory(object): def can_download(self, sd_validator): return True - def make_downloader(self, metadata, data_rate, payment_rate_manager, **kwargs): + def make_downloader(self, metadata, options, payment_rate_manager, **kwargs): + assert len(options) == 1 + data_rate = options[0] payment_rate_manager.min_blob_data_payment_rate = data_rate def save_source_if_blob(stream_hash): diff --git a/lbrynet/lbryfile/client/EncryptedFileOptions.py b/lbrynet/lbryfile/client/EncryptedFileOptions.py index f71457dfd..bc1d4ad99 100644 --- a/lbrynet/lbryfile/client/EncryptedFileOptions.py +++ b/lbrynet/lbryfile/client/EncryptedFileOptions.py @@ -43,17 +43,5 @@ class EncryptedFileOptions(object): prm.base.min_blob_data_payment_rate, get_default_data_rate_description() ), - DownloadOption( - [ - DownloadOptionChoice(bool, - None, - None, - bool_options_description=("Allow", "Disallow")), - ], - "Allow reuploading data downloaded for this file", - "allow upload", - True, - "Allow" - ), ] return options diff --git a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py index b257f4994..356cb02df 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py +++ b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py @@ -149,8 +149,10 @@ class ManagedEncryptedFileDownloaderFactory(object): return True @defer.inlineCallbacks - def make_downloader(self, metadata, data_rate, payment_rate_manager, download_directory=None, + def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, file_name=None): + assert len(options) == 1 + data_rate = options[0] stream_hash = yield save_sd_info(self.lbry_file_manager.stream_info_manager, metadata.validator.raw_info) if metadata.metadata_source == StreamMetadata.FROM_BLOB: diff --git a/lbrynet/lbryfilemanager/EncryptedFileManager.py b/lbrynet/lbryfilemanager/EncryptedFileManager.py index d00862b10..89900f01d 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileManager.py +++ b/lbrynet/lbryfilemanager/EncryptedFileManager.py @@ -269,9 +269,9 @@ class EncryptedFileManager(object): @rerun_if_locked def _save_lbry_file(self, stream_hash, data_payment_rate): def do_save(db_transaction): - db_transaction.execute("insert into lbry_file_options values (?, ?, ?)", - (data_payment_rate, ManagedEncryptedFileDownloader.STATUS_STOPPED, - stream_hash)) + row = (data_payment_rate, ManagedEncryptedFileDownloader.STATUS_STOPPED, stream_hash) + log.debug('Saving %s to db', row) + db_transaction.execute("insert into lbry_file_options values (?, ?, ?)", row) return db_transaction.lastrowid return self.sql_db.runInteraction(do_save) diff --git a/lbrynet/lbrylive/client/LiveStreamDownloader.py b/lbrynet/lbrylive/client/LiveStreamDownloader.py index 4b152f22a..913aba7dc 100644 --- a/lbrynet/lbrylive/client/LiveStreamDownloader.py +++ b/lbrynet/lbrylive/client/LiveStreamDownloader.py @@ -141,7 +141,7 @@ class FullLiveStreamDownloaderFactory(object): def can_download(self, sd_validator): return True - def make_downloader(self, metadata, data_rate, payment_rate_manager): + def make_downloader(self, metadata, options, payment_rate_manager): # TODO: check options for payment rate manager parameters prm = LiveStreamPaymentRateManager(self.default_payment_rate_manager, payment_rate_manager) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index d3224cda4..c184cb3d4 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -176,7 +176,7 @@ class AlwaysSend(object): @defer.inlineCallbacks def calculate_available_blob_size(blob_manager): blob_hashes = yield blob_manager.get_all_verified_blobs() - blobs = yield defer.DeferredList([blob_manager.get_blob(b, True) for b in blob_hashes]) + blobs = yield defer.DeferredList([blob_manager.get_blob(b) for b in blob_hashes]) defer.returnValue(sum(b.length for success, b in blobs if success and b.length)) diff --git a/lbrynet/lbrynet_daemon/Downloader.py b/lbrynet/lbrynet_daemon/Downloader.py index 0db02ee44..9202c22f9 100644 --- a/lbrynet/lbrynet_daemon/Downloader.py +++ b/lbrynet/lbrynet_daemon/Downloader.py @@ -119,9 +119,11 @@ class GetStream(object): @defer.inlineCallbacks def get_downloader(self, factory, stream_metadata): + # TODO: we should use stream_metadata.options.get_downloader_options + # instead of hard-coding the options to be [self.data_rate] downloader = yield factory.make_downloader( stream_metadata, - self.data_rate, + [self.data_rate], self.payment_rate_manager, download_directory=self.download_directory, file_name=self.file_name diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index b67ca1b1a..eb39e6bc4 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -113,7 +113,7 @@ class EncryptedFileReflectorClient(Protocol): def get_blobs(blobs): for (blob, _, _, blob_len) in blobs: if blob: - yield self.blob_manager.get_blob(blob, True, blob_len) + yield self.blob_manager.get_blob(blob, blob_len) dl = defer.DeferredList(list(get_blobs(blobs_in_stream)), consumeErrors=True) dl.addCallback(lambda blobs: [blob for r, blob in blobs if r and blob.is_validated()]) @@ -155,7 +155,7 @@ class EncryptedFileReflectorClient(Protocol): self.stream_descriptor = sd_blob d = self.factory.stream_info_manager.get_sd_blob_hashes_for_stream(self.factory.stream_hash) - d.addCallback(lambda sd: self.factory.blob_manager.get_blob(sd[0], True)) + d.addCallback(lambda sd: self.factory.blob_manager.get_blob(sd[0])) d.addCallback(_save_descriptor_blob) return d diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index 83f04f864..86e143fd5 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -222,7 +222,7 @@ class ReflectorServer(Protocol): sd_blob_size = request_dict[SD_BLOB_SIZE] if self.blob_write is None: - d = self.blob_manager.get_blob(sd_blob_hash, True, sd_blob_size) + d = self.blob_manager.get_blob(sd_blob_hash, sd_blob_size) d.addCallback(self.get_descriptor_response) d.addCallback(self.send_response) else: @@ -266,7 +266,7 @@ class ReflectorServer(Protocol): for blob in sd_blob['blobs']: if 'blob_hash' in blob and 'length' in blob: blob_hash, blob_len = blob['blob_hash'], blob['length'] - d = self.blob_manager.get_blob(blob_hash, True, blob_len) + d = self.blob_manager.get_blob(blob_hash, blob_len) d.addCallback(lambda blob: blob_hash if not blob.is_validated() else None) yield d diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index 1b028ae38..fef270b03 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -737,7 +737,7 @@ class TestTransfer(TestCase): factories = metadata.factories chosen_options = [ o.default_value for o in options.get_downloader_options(info_validator, prm)] - return factories[0].make_downloader(metadata, chosen_options, prm) + return factories[0].make_downloader(metadata, choosen_options, prm) def start_lbry_file(lbry_file): lbry_file = lbry_file diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index 7067d9776..2ed14f214 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -165,7 +165,7 @@ class TestReflector(unittest.TestCase): return defer.DeferredList(check_blob_ds) def verify_have_blob(blob_hash, blob_size): - d = self.server_blob_manager.get_blob(blob_hash, True) + d = self.server_blob_manager.get_blob(blob_hash) d.addCallback(lambda blob: verify_blob_completed(blob, blob_size)) return d @@ -196,7 +196,7 @@ class TestReflector(unittest.TestCase): return defer.DeferredList(check_blob_ds) def verify_have_blob(blob_hash, blob_size): - d = self.server_blob_manager.get_blob(blob_hash, True) + d = self.server_blob_manager.get_blob(blob_hash) d.addCallback(lambda blob: verify_blob_completed(blob, blob_size)) return d @@ -226,7 +226,7 @@ class TestReflector(unittest.TestCase): return defer.DeferredList(check_blob_ds) def verify_have_blob(blob_hash, blob_size): - d = self.server_blob_manager.get_blob(blob_hash, True) + d = self.server_blob_manager.get_blob(blob_hash) d.addCallback(lambda blob: verify_blob_completed(blob, blob_size)) return d From f7c105fd804bc75940fe20ee2b387486dcd2c8eb Mon Sep 17 00:00:00 2001 From: jobevers Date: Tue, 14 Feb 2017 13:30:14 -0600 Subject: [PATCH 7/8] fixes --- lbrynet/lbryfilemanager/EncryptedFileManager.py | 1 - lbrynet/lbrylive/client/LiveStreamDownloader.py | 2 +- tests/functional/test_misc.py | 17 +++++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lbrynet/lbryfilemanager/EncryptedFileManager.py b/lbrynet/lbryfilemanager/EncryptedFileManager.py index 89900f01d..be08cc8e8 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileManager.py +++ b/lbrynet/lbryfilemanager/EncryptedFileManager.py @@ -270,7 +270,6 @@ class EncryptedFileManager(object): def _save_lbry_file(self, stream_hash, data_payment_rate): def do_save(db_transaction): row = (data_payment_rate, ManagedEncryptedFileDownloader.STATUS_STOPPED, stream_hash) - log.debug('Saving %s to db', row) db_transaction.execute("insert into lbry_file_options values (?, ?, ?)", row) return db_transaction.lastrowid return self.sql_db.runInteraction(do_save) diff --git a/lbrynet/lbrylive/client/LiveStreamDownloader.py b/lbrynet/lbrylive/client/LiveStreamDownloader.py index 913aba7dc..a59eff1cf 100644 --- a/lbrynet/lbrylive/client/LiveStreamDownloader.py +++ b/lbrynet/lbrylive/client/LiveStreamDownloader.py @@ -160,7 +160,7 @@ class FullLiveStreamDownloaderFactory(object): def create_downloader(stream_hash): stream_downloader = FullLiveStreamDownloader(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.stream_info_manager, - prm, self.wallet, True) + prm, self.wallet) d = stream_downloader.set_stream_info() d.addCallback(lambda _: stream_downloader) return d diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index fef270b03..a9ef36d08 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -1,5 +1,3 @@ -from lbrynet.core import log_support - import io import logging from multiprocessing import Process, Event, Queue @@ -61,7 +59,6 @@ GenFile = mocks.GenFile test_create_stream_sd_file = mocks.create_stream_sd_file DummyBlobAvailabilityTracker = mocks.BlobAvailabilityTracker -log = logging.getLogger(__name__) log_format = "%(funcName)s(): %(message)s" logging.basicConfig(level=logging.WARNING, format=log_format) @@ -142,7 +139,11 @@ class LbryUploader(object): d.addCallback(lambda _: self.create_stream()) d.addCallback(self.create_stream_descriptor) d.addCallback(self.put_sd_hash_on_queue) - d.addErrback(log.fail(), 'Failed to start') + + def print_error(err): + logging.critical("Server error: %s", err.getErrorMessage()) + + d.addErrback(print_error) return d def start_server(self): @@ -472,7 +473,11 @@ def start_blob_uploader(blob_hash_queue, kill_event, dead_event, slow, is_genero d.addCallback(lambda _: start_server()) d.addCallback(lambda _: create_single_blob()) d.addCallback(put_blob_hash_on_queue) - d.addErrback(log.fail(), 'Failed to start') + + def print_error(err): + logging.critical("Server error: %s", err.getErrorMessage()) + + d.addErrback(print_error) return d def start_server(): @@ -737,7 +742,7 @@ class TestTransfer(TestCase): factories = metadata.factories chosen_options = [ o.default_value for o in options.get_downloader_options(info_validator, prm)] - return factories[0].make_downloader(metadata, choosen_options, prm) + return factories[0].make_downloader(metadata, chosen_options, prm) def start_lbry_file(lbry_file): lbry_file = lbry_file From 1fd87b7caa7d1bd4a95e6ca2b6208ca8ed4d8abb Mon Sep 17 00:00:00 2001 From: jobevers Date: Tue, 14 Feb 2017 13:31:15 -0600 Subject: [PATCH 8/8] add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eade334dc..b008aaf3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ can and probably will change functionality and break backwards compatability at anytime. ## [Unreleased] +### Changed + * Remove unused upload_allowed option + ### Fixed * Add better logging to help track down [#478](https://github.com/lbryio/lbry/issues/478)