Merge pull request #446 from lbryio/reseed

Script to Reseed a File

lbrynet/core/HashBlob.py

@@ -242,6 +242,7 @@ class BlobFile(HashBlob):
                 self.readers += 1
                 return file_handle
             except IOError:
+                log.exception('Failed to open %s', self.file_path)
                 self.close_read_handle(file_handle)
         return None
 

lbrynet/core/StreamCreator.py

@@ -64,6 +64,8 @@ class StreamCreator(object):
     def _finished(self):
         pass
 
+    # TODO: move the stream creation process to its own thread and
+    #       remove the reactor from this process.
     def write(self, data):
         from twisted.internet import reactor
         self._write(data)

lbrynet/lbryfilemanager/EncryptedFileCreator.py

@@ -77,6 +77,12 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
         return d
 
 
+# TODO: this should be run its own thread. Encrypting a large file can
+#       be very cpu intensive and there is no need to run that on the
+#       main reactor thread. The FileSender mechanism that is used is
+#       great when sending over the network, but this is all local so
+#       we can simply read the file from the disk without needing to
+#       involve reactor.
 def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None,
                      iv_generator=None, suggested_file_name=None):
     """Turn a plain file into an LBRY File.
@@ -146,6 +152,10 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
         suggested_file_name)
 
     def start_stream():
+        # TODO: Using FileSender isn't necessary, we can just read
+        #       straight from the disk. The stream creation process
+        #       should be in its own thread anyway so we don't need to
+        #       worry about interacting with the twisted reactor
         file_sender = FileSender()
         d = file_sender.beginFileTransfer(file_handle, lbry_file_creator)
         d.addCallback(lambda _: stop_file(lbry_file_creator))

lbrynet/lbryfilemanager/EncryptedFileManager.py

@@ -41,6 +41,7 @@ class EncryptedFileManager(object):
     def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
         self.session = session
         self.stream_info_manager = stream_info_manager
+        # TODO: why is sd_identifier part of the file manager?
         self.sd_identifier = sd_identifier
         self.lbry_files = []
         self.sql_db = None

scripts/decrypt_blob.py

@@ -0,0 +1,57 @@
+"""Decrypt a single blob"""
+import argparse
+import binascii
+import logging
+import os
+import sys
+
+from twisted.internet import defer
+from twisted.internet import reactor
+
+from lbrynet import conf
+from lbrynet.cryptstream import CryptBlob
+from lbrynet.core import HashBlob
+from lbrynet.core import log_support
+
+
+log = logging.getLogger('decrypt_blob')
+
+
+def main():
+    conf.initialize_settings()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('blob_file')
+    parser.add_argument('hex_key')
+    parser.add_argument('hex_iv')
+    parser.add_argument('output')
+    args = parser.parse_args()
+    log_support.configure_console()
+
+    d = run(args)
+    reactor.run()
+
+
+@defer.inlineCallbacks
+def run(args):
+    try:
+        yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output)
+    except Exception:
+        log.exception('Failed to decrypt blob')
+    finally:
+        reactor.callLater(0, reactor.stop)
+
+
+@defer.inlineCallbacks
+def decrypt_blob(blob_file, key, iv, output):
+    filename = os.path.abspath(blob_file)
+    length = os.path.getsize(filename)
+    directory, blob_hash = os.path.split(filename)
+    blob = HashBlob.BlobFile(directory, blob_hash, True, length)
+    decryptor = CryptBlob.StreamBlobDecryptor(
+        blob, binascii.unhexlify(key), binascii.unhexlify(iv), length)
+    with open(output, 'w') as f:
+        yield decryptor.decrypt(f.write)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/encrypt_blob.py

@@ -0,0 +1,71 @@
+"""Encrypt a single file using the given key and iv"""
+import argparse
+import binascii
+import logging
+import os
+import StringIO
+import sys
+
+from twisted.internet import defer
+from twisted.internet import reactor
+
+from lbrynet import conf
+from lbrynet.cryptstream import CryptBlob
+from lbrynet.core import HashBlob
+from lbrynet.core import log_support
+from lbrynet.core import cryptoutils
+
+
+log = logging.getLogger('decrypt_blob')
+
+
+def main():
+    conf.initialize_settings()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filename')
+    parser.add_argument('hex_key')
+    parser.add_argument('hex_iv')
+    args = parser.parse_args()
+    log_support.configure_console(level='DEBUG')
+
+    d = run(args)
+    reactor.run()
+
+
+@defer.inlineCallbacks
+def run(args):
+    try:
+        yield encrypt_blob(args.filename, args.hex_key, args.hex_iv)
+    except Exception:
+        log.exception('Failed to encrypt blob')
+    finally:
+        reactor.callLater(0, reactor.stop)
+
+
+def encrypt_blob(filename, key, iv):
+    blob = Blob()
+    blob_maker = CryptBlob.CryptStreamBlobMaker(
+        binascii.unhexlify(key), binascii.unhexlify(iv), 0, blob)
+    with open(filename) as fin:
+        blob_maker.write(fin.read())
+    blob_maker.close()
+
+
+class Blob(object):
+    def __init__(self):
+        self.data = StringIO.StringIO()
+
+    def write(self, data):
+        self.data.write(data)
+
+    def close(self):
+        hashsum = cryptoutils.get_lbry_hash_obj()
+        buffer = self.data.getvalue()
+        hashsum.update(buffer)
+        with open(hashsum.hexdigest(), 'w') as fout:
+            fout.write(buffer)
+        return defer.succeed(True)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/query_available_blobs.py

@@ -22,7 +22,6 @@ from lbrynet import conf
 from lbrynet.core import Error
 from lbrynet.core import Wallet
 from lbrynet.core import BlobAvailability
-from lbrynet.core import BlobManager
 from lbrynet.core import HashAnnouncer
 from lbrynet.core import PeerManager
 from lbrynet.core import Session

scripts/reseed_file.py

@@ -0,0 +1,94 @@
+"""Reseed a file.
+
+Given a file and a matching sd_blob,
+re-chunk and encrypt the file, adding
+the new blobs to the manager.
+"""
+import argparse
+import binascii
+import logging
+import json
+import os
+import sys
+
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.protocols import basic
+
+from lbrynet import conf
+from lbrynet.core import BlobManager
+from lbrynet.core import HashAnnouncer
+from lbrynet.core import log_support
+from lbrynet.cryptstream import CryptStreamCreator
+
+
+log = logging.getLogger('reseed_file')
+
+
+def main():
+    conf.initialize_settings()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_file')
+    parser.add_argument('sd_blob', help='a json file containing a key and the IVs')
+    args = parser.parse_args()
+    log_support.configure_console()
+
+    run(args)
+    reactor.run()
+
+
+@defer.inlineCallbacks
+def run(args):
+    try:
+        yield reseed_file(args.input_file, args.sd_blob)
+    except Exception as e:
+        log.exception('Failed to reseed')
+    finally:
+        reactor.stop()
+
+
+@defer.inlineCallbacks
+def reseed_file(input_file, sd_blob):
+    sd_blob = SdBlob.new_instance(sd_blob)
+    db_dir = conf.settings['data_dir']
+    blobfile_dir = os.path.join(db_dir, "blobfiles")
+    announcer = HashAnnouncer.DummyHashAnnouncer()
+    blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir)
+    yield blob_manager.setup()
+    creator = CryptStreamCreator.CryptStreamCreator(
+        blob_manager, None, sd_blob.key(), sd_blob.iv_generator())
+    file_sender = basic.FileSender()
+    with open(input_file) as f:
+        yield file_sender.beginFileTransfer(f, creator)
+        yield creator.stop()
+    for blob_info in sd_blob.blob_infos():
+        if 'blob_hash' not in blob_info:
+            # the last blob is always empty and without a hash
+            continue
+        blob = yield blob_manager.get_blob(blob_info['blob_hash'], True)
+        if not blob.verified:
+            print "Blob {} is not verified".format(blob)
+
+
+class SdBlob(object):
+    def __init__(self, contents):
+        self.contents = contents
+
+    def key(self):
+        return binascii.unhexlify(self.contents['key'])
+
+    def iv_generator(self):
+        for blob_info in self.blob_infos():
+            yield binascii.unhexlify(blob_info['iv'])
+
+    def blob_infos(self):
+        return self.contents['blobs']
+
+    @classmethod
+    def new_instance(cls, filename):
+        with open(filename) as f:
+            return cls(json.load(f))
+
+
+if __name__ == '__main__':
+    sys.exit(main())