more specific exceptions, and change when blobs are deleted

More specific exceptions are raised when problems are encountered
looking up metadata and validating stream descriptor files, and
on the GUI those more specific exceptions are used to prevent
errors from being presented to the user.

If the user has selected the option to re-upload data for some
stream, blobs downloaded for that purpose will not be deleted
when they are finished being output. Instead, by default, for
the GUI they will be deleted when the stream is removed from
the GUI. That can be changed so they are not deleted at all,
using the lbry.conf file.

lbrynet/core/Error.py

@@ -26,11 +26,29 @@ class UnknownNameError(Exception):
     def __init__(self, name):
         self.name = name
 
+    def __str__(self):
+        return repr(self.name)
+
+
+class UnknownStreamTypeError(Exception):
+    def __init__(self, stream_type):
+        self.stream_type = stream_type
+
+    def __str__(self):
+        return repr(self.stream_type)
+
+
+class InvalidStreamDescriptorError(Exception):
+    pass
+
 
 class InvalidStreamInfoError(Exception):
     def __init__(self, name):
         self.name = name
 
+    def __str__(self):
+        return repr(self.name)
+
 
 class MisbehavingPeerError(Exception):
     pass

lbrynet/core/StreamDescriptor.py

@@ -3,6 +3,7 @@ import json
 import logging
 from twisted.internet import threads
 from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
+from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
 
 
 class StreamDescriptorReader(object):
@@ -165,19 +166,23 @@ class StreamDescriptorIdentifier(object):
         return d
 
     def _get_factories(self, stream_type):
-        assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type)
+        if not stream_type in self._stream_downloader_factories:
+            raise UnknownStreamTypeError(stream_type)
         return self._stream_downloader_factories[stream_type]
 
     def _get_validator(self, stream_type):
-        assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
+        if not stream_type in self._stream_downloader_factories:
+            raise UnknownStreamTypeError(stream_type)
         return self._sd_info_validators[stream_type]
 
     def _get_options(self, stream_type):
-        assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
+        if not stream_type in self._stream_downloader_factories:
+            raise UnknownStreamTypeError(stream_type)
         return self._stream_options[stream_type]
 
     def _return_info_and_factories(self, sd_info):
-        assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.'
+        if not 'stream_type' in sd_info:
+            raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
         stream_type = sd_info['stream_type']
         validator = self._get_validator(stream_type)(sd_info)
         factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]

lbrynet/lbryfile/StreamDescriptor.py

@@ -3,7 +3,7 @@ import logging
 from lbrynet.core.cryptoutils import get_lbry_hash_obj
 from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
 from twisted.internet import defer
-from lbrynet.core.Error import DuplicateStreamHashError
+from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
 
 
 LBRYFileStreamType = "lbryfile"
@@ -88,11 +88,10 @@ class LBRYFileStreamDescriptorValidator(object):
             stream_hash = self.raw_info['stream_hash']
             blobs = self.raw_info['blobs']
         except KeyError as e:
-            raise ValueError("Invalid stream descriptor. Missing '%s'" % (e.args[0]))
+            raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
         for c in hex_suggested_file_name:
             if c not in '0123456789abcdef':
-                raise ValueError("Invalid stream descriptor: "
+                raise InvalidStreamDescriptorError("Suggested file name is not a hex-encoded string")
-                                 "suggested file name is not a hex-encoded string")
         h = get_lbry_hash_obj()
         h.update(hex_stream_name)
         h.update(key)
@@ -118,10 +117,10 @@ class LBRYFileStreamDescriptorValidator(object):
         for blob in blobs:
             blobs_hashsum.update(get_blob_hashsum(blob))
         if blobs[-1]['length'] != 0:
-            raise ValueError("Improperly formed stream descriptor. Must end with a zero-length blob.")
+            raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
         h.update(blobs_hashsum.digest())
         if h.hexdigest() != stream_hash:
-            raise ValueError("Stream hash does not match stream metadata")
+            raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
         return defer.succeed(True)
 
     def info_to_show(self):

lbrynet/lbryfile/client/LBRYFileDownloader.py

@@ -146,7 +146,7 @@ class LBRYFileSaver(LBRYFileDownloader):
 
     def _get_progress_manager(self, download_manager):
         return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
-                                         delete_blob_after_finished=True)
+                                         delete_blob_after_finished=not self.upload_allowed)
 
     def _setup_output(self):
         def open_file():
@@ -216,7 +216,7 @@ class LBRYFileOpener(LBRYFileDownloader):
 
     def _get_progress_manager(self, download_manager):
         return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
-                                         delete_blob_after_finished=True)
+                                         delete_blob_after_finished=not self.upload_allowed)
 
     def _setup_output(self):
         def start_process():

lbrynet/lbrylive/StreamDescriptor.py

@@ -2,7 +2,7 @@ import binascii
 import logging
 from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
 from twisted.internet import defer, threads
-from lbrynet.core.Error import DuplicateStreamHashError
+from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
 from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
 from lbrynet.interfaces import IStreamDescriptorValidator
 from zope.interface import implements
@@ -98,7 +98,7 @@ class LBRYLiveStreamDescriptorValidator(object):
         h.update(public_key)
         h.update(key)
         if h.hexdigest() != stream_hash:
-            raise ValueError("Stream hash does not match stream metadata")
+            raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
         blobs = self.raw_info['blobs']
 
         def check_blob_signatures():
@@ -121,7 +121,7 @@ class LBRYLiveStreamDescriptorValidator(object):
                 hashsum.update(iv)
                 hashsum.update(str(length))
                 if not verify_signature(hashsum.digest(), signature, public_key):
-                    raise ValueError("Invalid signature in stream descriptor")
+                    raise InvalidStreamDescriptorError("Invalid signature in stream descriptor")
 
         return threads.deferToThread(check_blob_signatures)
 

lbrynet/lbrynet_downloader_gui/LBRYDownloader.py

@@ -4,6 +4,8 @@ import tkMessageBox
 from Crypto import Random
 from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE
 from lbrynet.core import StreamDescriptor
+from lbrynet.core.Error import UnknownNameError, UnknownStreamTypeError, InvalidStreamDescriptorError
+from lbrynet.core.Error import InvalidStreamInfoError
 from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet
 from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.core.Session import LBRYSession
@@ -46,6 +48,7 @@ class LBRYDownloader(object):
         self.default_blob_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE
         self.use_upnp = False
         self.start_lbrycrdd = True
+        self.delete_blobs_on_remove = True
         self.blob_request_payment_rate_manager = None
 
     def start(self):
@@ -175,6 +178,13 @@ class LBRYDownloader(object):
                     elif field_name == "download_directory":
                         logging.debug("Setting download_directory to %s", str(field_value))
                         self.download_directory = field_value
+                    elif field_name == "delete_blobs_on_stream_remove":
+                        if field_value.lower() == "true":
+                            self.delete_blobs_on_remove = True
+                        elif field_value.lower() == "false":
+                            self.delete_blobs_on_remove = False
+                        else:
+                            raise ValueError("delete_blobs_on_stream_remove must be set to True or False")
                     else:
                         logging.warning("Got unknown configuration field: %s", field_name)
 
@@ -312,7 +322,7 @@ class LBRYDownloader(object):
         def get_sd_hash(value):
             if 'stream_hash' in value:
                 return value['stream_hash']
-            raise ValueError("Invalid stream")
+            raise UnknownNameError(uri)
 
         def get_sd_blob(sd_hash):
             stream_frame.show_metadata_status("name resolved, fetching metadata...")
@@ -393,6 +403,7 @@ class LBRYDownloader(object):
             return arg
 
         def start_download(downloader):
+            stream_frame.stream_hash = downloader.stream_hash
             l = task.LoopingCall(show_stream_status, downloader)
             l.start(1)
             d = downloader.start()
@@ -424,8 +435,38 @@ class LBRYDownloader(object):
             logging.error(err.getErrorMessage())
             stream_frame.show_download_done(payment_rate_manager.points_paid)
 
-        resolve_d.addErrback(lambda err: err.trap(defer.CancelledError))
+        resolve_d.addErrback(lambda err: err.trap(defer.CancelledError, UnknownNameError,
+                                                  UnknownStreamTypeError, InvalidStreamDescriptorError,
+                                                  InvalidStreamInfoError))
         resolve_d.addErrback(show_err)
+
+        def delete_associated_blobs():
+            if stream_frame.stream_hash is None or self.delete_blobs_on_remove is False:
+                return defer.succeed(True)
+            d1 = self.stream_info_manager.get_blobs_for_stream(stream_frame.stream_hash)
+
+            def get_blob_hashes(blob_infos):
+                return [b[0] for b in blob_infos if b[0] is not None]
+
+            d1.addCallback(get_blob_hashes)
+            d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(stream_frame.stream_hash)
+
+            def combine_blob_hashes(results):
+                blob_hashes = []
+                for success, result in results:
+                    if success is True:
+                        blob_hashes.extend(result)
+                return blob_hashes
+
+            def delete_blobs(blob_hashes):
+                return self.session.blob_manager.delete_blobs(blob_hashes)
+
+            dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
+            dl.addCallback(combine_blob_hashes)
+            dl.addCallback(delete_blobs)
+            return dl
+
+        resolve_d.addCallback(lambda _: delete_associated_blobs())
         self._add_download_deferred(resolve_d, stream_frame)
 
     def _add_download_deferred(self, d, stream_frame):

lbrynet/lbrynet_downloader_gui/StreamFrame.py

@@ -10,6 +10,7 @@ class StreamFrame(object):
     def __init__(self, app, uri):
         self.app = app
         self.uri = uri
+        self.stream_hash = None
         self.cancel_func = None
 
         self.stream_frame = ttk.Frame(self.app.streams_frame, style="B.TFrame")

lbrynet/lbrynet_downloader_gui/lbry.conf

@@ -94,4 +94,16 @@
 # to download encrypted chunks of data from this application,
 # if this application is acting as a server.
 #
-# peer_port = 3333
+# peer_port = 3333
+
+
+# ===== delete_blobs_on_stream_remove =====
+# If this is set to True, all blobs associated with the stream
+# will be deleted when the stream is removed from the GUI,
+# whether by clicking the 'x' or by closing the application.
+# If this is set to False, they will not be deleted then.
+# However, they may be deleted otherwise. For example if the
+# option to allow reuploading is set to False, they will be
+# deleted as soon as they're outputted by the application.
+#
+# delete_blobs_on_stream_remove = True