From 192ac6959acc244e9e4a36f0dbbedf0f764af0dc Mon Sep 17 00:00:00 2001 From: Jimmy Kiselak Date: Tue, 1 Sep 2015 17:49:26 -0400 Subject: [PATCH] more specific exceptions, and change when blobs are deleted More specific exceptions are raised when problems are encountered looking up metadata and validating stream descriptor files, and on the GUI those more specific exceptions are used to prevent errors from being presented to the user. If the user has selected the option to re-upload data for some stream, blobs downloaded for that purpose will not be deleted when they are finished being output. Instead, by default, for the GUI they will be deleted when the stream is removed from the GUI. That can be changed so they are not deleted at all, using the lbry.conf file. --- lbrynet/core/Error.py | 18 ++++++++ lbrynet/core/StreamDescriptor.py | 13 ++++-- lbrynet/lbryfile/StreamDescriptor.py | 11 +++-- lbrynet/lbryfile/client/LBRYFileDownloader.py | 4 +- lbrynet/lbrylive/StreamDescriptor.py | 6 +-- .../lbrynet_downloader_gui/LBRYDownloader.py | 45 ++++++++++++++++++- lbrynet/lbrynet_downloader_gui/StreamFrame.py | 1 + lbrynet/lbrynet_downloader_gui/lbry.conf | 14 +++++- 8 files changed, 94 insertions(+), 18 deletions(-) diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index 3780998cb..08c03ce05 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -26,11 +26,29 @@ class UnknownNameError(Exception): def __init__(self, name): self.name = name + def __str__(self): + return repr(self.name) + + +class UnknownStreamTypeError(Exception): + def __init__(self, stream_type): + self.stream_type = stream_type + + def __str__(self): + return repr(self.stream_type) + + +class InvalidStreamDescriptorError(Exception): + pass + class InvalidStreamInfoError(Exception): def __init__(self, name): self.name = name + def __str__(self): + return repr(self.name) + class MisbehavingPeerError(Exception): pass diff --git a/lbrynet/core/StreamDescriptor.py b/lbrynet/core/StreamDescriptor.py index 04ed216ff..a2d72f54b 100644 --- a/lbrynet/core/StreamDescriptor.py +++ b/lbrynet/core/StreamDescriptor.py @@ -3,6 +3,7 @@ import json import logging from twisted.internet import threads from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader +from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError class StreamDescriptorReader(object): @@ -165,19 +166,23 @@ class StreamDescriptorIdentifier(object): return d def _get_factories(self, stream_type): - assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type) + if not stream_type in self._stream_downloader_factories: + raise UnknownStreamTypeError(stream_type) return self._stream_downloader_factories[stream_type] def _get_validator(self, stream_type): - assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type) + if not stream_type in self._stream_downloader_factories: + raise UnknownStreamTypeError(stream_type) return self._sd_info_validators[stream_type] def _get_options(self, stream_type): - assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type) + if not stream_type in self._stream_downloader_factories: + raise UnknownStreamTypeError(stream_type) return self._stream_options[stream_type] def _return_info_and_factories(self, sd_info): - assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.' + if not 'stream_type' in sd_info: + raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.') stream_type = sd_info['stream_type'] validator = self._get_validator(stream_type)(sd_info) factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)] diff --git a/lbrynet/lbryfile/StreamDescriptor.py b/lbrynet/lbryfile/StreamDescriptor.py index c1244c086..5872b5714 100644 --- a/lbrynet/lbryfile/StreamDescriptor.py +++ b/lbrynet/lbryfile/StreamDescriptor.py @@ -3,7 +3,7 @@ import logging from lbrynet.core.cryptoutils import get_lbry_hash_obj from lbrynet.cryptstream.CryptBlob import CryptBlobInfo from twisted.internet import defer -from lbrynet.core.Error import DuplicateStreamHashError +from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError LBRYFileStreamType = "lbryfile" @@ -88,11 +88,10 @@ class LBRYFileStreamDescriptorValidator(object): stream_hash = self.raw_info['stream_hash'] blobs = self.raw_info['blobs'] except KeyError as e: - raise ValueError("Invalid stream descriptor. Missing '%s'" % (e.args[0])) + raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0])) for c in hex_suggested_file_name: if c not in '0123456789abcdef': - raise ValueError("Invalid stream descriptor: " - "suggested file name is not a hex-encoded string") + raise InvalidStreamDescriptorError("Suggested file name is not a hex-encoded string") h = get_lbry_hash_obj() h.update(hex_stream_name) h.update(key) @@ -118,10 +117,10 @@ class LBRYFileStreamDescriptorValidator(object): for blob in blobs: blobs_hashsum.update(get_blob_hashsum(blob)) if blobs[-1]['length'] != 0: - raise ValueError("Improperly formed stream descriptor. Must end with a zero-length blob.") + raise InvalidStreamDescriptorError("Does not end with a zero-length blob.") h.update(blobs_hashsum.digest()) if h.hexdigest() != stream_hash: - raise ValueError("Stream hash does not match stream metadata") + raise InvalidStreamDescriptorError("Stream hash does not match stream metadata") return defer.succeed(True) def info_to_show(self): diff --git a/lbrynet/lbryfile/client/LBRYFileDownloader.py b/lbrynet/lbryfile/client/LBRYFileDownloader.py index e8bbe1921..3de1a3898 100644 --- a/lbrynet/lbryfile/client/LBRYFileDownloader.py +++ b/lbrynet/lbryfile/client/LBRYFileDownloader.py @@ -146,7 +146,7 @@ class LBRYFileSaver(LBRYFileDownloader): def _get_progress_manager(self, download_manager): return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager, - delete_blob_after_finished=True) + delete_blob_after_finished=not self.upload_allowed) def _setup_output(self): def open_file(): @@ -216,7 +216,7 @@ class LBRYFileOpener(LBRYFileDownloader): def _get_progress_manager(self, download_manager): return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager, - delete_blob_after_finished=True) + delete_blob_after_finished=not self.upload_allowed) def _setup_output(self): def start_process(): diff --git a/lbrynet/lbrylive/StreamDescriptor.py b/lbrynet/lbrylive/StreamDescriptor.py index 441c26ea4..211ccf00d 100644 --- a/lbrynet/lbrylive/StreamDescriptor.py +++ b/lbrynet/lbrylive/StreamDescriptor.py @@ -2,7 +2,7 @@ import binascii import logging from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature from twisted.internet import defer, threads -from lbrynet.core.Error import DuplicateStreamHashError +from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError from lbrynet.lbrylive.LiveBlob import LiveBlobInfo from lbrynet.interfaces import IStreamDescriptorValidator from zope.interface import implements @@ -98,7 +98,7 @@ class LBRYLiveStreamDescriptorValidator(object): h.update(public_key) h.update(key) if h.hexdigest() != stream_hash: - raise ValueError("Stream hash does not match stream metadata") + raise InvalidStreamDescriptorError("Stream hash does not match stream metadata") blobs = self.raw_info['blobs'] def check_blob_signatures(): @@ -121,7 +121,7 @@ class LBRYLiveStreamDescriptorValidator(object): hashsum.update(iv) hashsum.update(str(length)) if not verify_signature(hashsum.digest(), signature, public_key): - raise ValueError("Invalid signature in stream descriptor") + raise InvalidStreamDescriptorError("Invalid signature in stream descriptor") return threads.deferToThread(check_blob_signatures) diff --git a/lbrynet/lbrynet_downloader_gui/LBRYDownloader.py b/lbrynet/lbrynet_downloader_gui/LBRYDownloader.py index 702c9fc98..3d79bdd71 100644 --- a/lbrynet/lbrynet_downloader_gui/LBRYDownloader.py +++ b/lbrynet/lbrynet_downloader_gui/LBRYDownloader.py @@ -4,6 +4,8 @@ import tkMessageBox from Crypto import Random from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE from lbrynet.core import StreamDescriptor +from lbrynet.core.Error import UnknownNameError, UnknownStreamTypeError, InvalidStreamDescriptorError +from lbrynet.core.Error import InvalidStreamInfoError from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.Session import LBRYSession @@ -46,6 +48,7 @@ class LBRYDownloader(object): self.default_blob_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE self.use_upnp = False self.start_lbrycrdd = True + self.delete_blobs_on_remove = True self.blob_request_payment_rate_manager = None def start(self): @@ -175,6 +178,13 @@ class LBRYDownloader(object): elif field_name == "download_directory": logging.debug("Setting download_directory to %s", str(field_value)) self.download_directory = field_value + elif field_name == "delete_blobs_on_stream_remove": + if field_value.lower() == "true": + self.delete_blobs_on_remove = True + elif field_value.lower() == "false": + self.delete_blobs_on_remove = False + else: + raise ValueError("delete_blobs_on_stream_remove must be set to True or False") else: logging.warning("Got unknown configuration field: %s", field_name) @@ -312,7 +322,7 @@ class LBRYDownloader(object): def get_sd_hash(value): if 'stream_hash' in value: return value['stream_hash'] - raise ValueError("Invalid stream") + raise UnknownNameError(uri) def get_sd_blob(sd_hash): stream_frame.show_metadata_status("name resolved, fetching metadata...") @@ -393,6 +403,7 @@ class LBRYDownloader(object): return arg def start_download(downloader): + stream_frame.stream_hash = downloader.stream_hash l = task.LoopingCall(show_stream_status, downloader) l.start(1) d = downloader.start() @@ -424,8 +435,38 @@ class LBRYDownloader(object): logging.error(err.getErrorMessage()) stream_frame.show_download_done(payment_rate_manager.points_paid) - resolve_d.addErrback(lambda err: err.trap(defer.CancelledError)) + resolve_d.addErrback(lambda err: err.trap(defer.CancelledError, UnknownNameError, + UnknownStreamTypeError, InvalidStreamDescriptorError, + InvalidStreamInfoError)) resolve_d.addErrback(show_err) + + def delete_associated_blobs(): + if stream_frame.stream_hash is None or self.delete_blobs_on_remove is False: + return defer.succeed(True) + d1 = self.stream_info_manager.get_blobs_for_stream(stream_frame.stream_hash) + + def get_blob_hashes(blob_infos): + return [b[0] for b in blob_infos if b[0] is not None] + + d1.addCallback(get_blob_hashes) + d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(stream_frame.stream_hash) + + def combine_blob_hashes(results): + blob_hashes = [] + for success, result in results: + if success is True: + blob_hashes.extend(result) + return blob_hashes + + def delete_blobs(blob_hashes): + return self.session.blob_manager.delete_blobs(blob_hashes) + + dl = defer.DeferredList([d1, d2], fireOnOneErrback=True) + dl.addCallback(combine_blob_hashes) + dl.addCallback(delete_blobs) + return dl + + resolve_d.addCallback(lambda _: delete_associated_blobs()) self._add_download_deferred(resolve_d, stream_frame) def _add_download_deferred(self, d, stream_frame): diff --git a/lbrynet/lbrynet_downloader_gui/StreamFrame.py b/lbrynet/lbrynet_downloader_gui/StreamFrame.py index afeb1612f..785fb8cec 100644 --- a/lbrynet/lbrynet_downloader_gui/StreamFrame.py +++ b/lbrynet/lbrynet_downloader_gui/StreamFrame.py @@ -10,6 +10,7 @@ class StreamFrame(object): def __init__(self, app, uri): self.app = app self.uri = uri + self.stream_hash = None self.cancel_func = None self.stream_frame = ttk.Frame(self.app.streams_frame, style="B.TFrame") diff --git a/lbrynet/lbrynet_downloader_gui/lbry.conf b/lbrynet/lbrynet_downloader_gui/lbry.conf index 2b3a6110a..f165ad90f 100644 --- a/lbrynet/lbrynet_downloader_gui/lbry.conf +++ b/lbrynet/lbrynet_downloader_gui/lbry.conf @@ -94,4 +94,16 @@ # to download encrypted chunks of data from this application, # if this application is acting as a server. # -# peer_port = 3333 \ No newline at end of file +# peer_port = 3333 + + +# ===== delete_blobs_on_stream_remove ===== +# If this is set to True, all blobs associated with the stream +# will be deleted when the stream is removed from the GUI, +# whether by clicking the 'x' or by closing the application. +# If this is set to False, they will not be deleted then. +# However, they may be deleted otherwise. For example if the +# option to allow reuploading is set to False, they will be +# deleted as soon as they're outputted by the application. +# +# delete_blobs_on_stream_remove = True \ No newline at end of file