From 90d3a0ef9c54c501cedf4f8351844512a17975a6 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 20 Jan 2017 14:50:10 -0600 Subject: [PATCH] Add script to reseed a file --- lbrynet/core/HashBlob.py | 1 + lbrynet/core/StreamCreator.py | 2 + .../lbryfilemanager/EncryptedFileCreator.py | 10 ++ .../lbryfilemanager/EncryptedFileManager.py | 1 + scripts/decrypt_blob.py | 57 +++++++++++ scripts/encrypt_blob.py | 71 ++++++++++++++ scripts/query_available_blobs.py | 1 - scripts/reseed_file.py | 94 +++++++++++++++++++ 8 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 scripts/decrypt_blob.py create mode 100644 scripts/encrypt_blob.py create mode 100644 scripts/reseed_file.py diff --git a/lbrynet/core/HashBlob.py b/lbrynet/core/HashBlob.py index 929d5637e..f09d0709e 100644 --- a/lbrynet/core/HashBlob.py +++ b/lbrynet/core/HashBlob.py @@ -243,6 +243,7 @@ class BlobFile(HashBlob): self.readers += 1 return file_handle except IOError: + log.exception('Failed to open %s', self.file_path) self.close_read_handle(file_handle) return None diff --git a/lbrynet/core/StreamCreator.py b/lbrynet/core/StreamCreator.py index cc579865d..4aa0ae542 100644 --- a/lbrynet/core/StreamCreator.py +++ b/lbrynet/core/StreamCreator.py @@ -64,6 +64,8 @@ class StreamCreator(object): def _finished(self): pass + # TODO: move the stream creation process to its own thread and + # remove the reactor from this process. def write(self, data): from twisted.internet import reactor self._write(data) diff --git a/lbrynet/lbryfilemanager/EncryptedFileCreator.py b/lbrynet/lbryfilemanager/EncryptedFileCreator.py index 43369dbe5..69ae82fd0 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileCreator.py +++ b/lbrynet/lbryfilemanager/EncryptedFileCreator.py @@ -77,6 +77,12 @@ class EncryptedFileStreamCreator(CryptStreamCreator): return d +# TODO: this should be run its own thread. Encrypting a large file can +# be very cpu intensive and there is no need to run that on the +# main reactor thread. The FileSender mechanism that is used is +# great when sending over the network, but this is all local so +# we can simply read the file from the disk without needing to +# involve reactor. def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None, iv_generator=None, suggested_file_name=None): """Turn a plain file into an LBRY File. @@ -146,6 +152,10 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non suggested_file_name) def start_stream(): + # TODO: Using FileSender isn't necessary, we can just read + # straight from the disk. The stream creation process + # should be in its own thread anyway so we don't need to + # worry about interacting with the twisted reactor file_sender = FileSender() d = file_sender.beginFileTransfer(file_handle, lbry_file_creator) d.addCallback(lambda _: stop_file(lbry_file_creator)) diff --git a/lbrynet/lbryfilemanager/EncryptedFileManager.py b/lbrynet/lbryfilemanager/EncryptedFileManager.py index b4ad6b7c4..7791678ab 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileManager.py +++ b/lbrynet/lbryfilemanager/EncryptedFileManager.py @@ -41,6 +41,7 @@ class EncryptedFileManager(object): def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None): self.session = session self.stream_info_manager = stream_info_manager + # TODO: why is sd_identifier part of the file manager? self.sd_identifier = sd_identifier self.lbry_files = [] self.sql_db = None diff --git a/scripts/decrypt_blob.py b/scripts/decrypt_blob.py new file mode 100644 index 000000000..bc905bf2e --- /dev/null +++ b/scripts/decrypt_blob.py @@ -0,0 +1,57 @@ +"""Decrypt a single blob""" +import argparse +import binascii +import logging +import os +import sys + +from twisted.internet import defer +from twisted.internet import reactor + +from lbrynet import conf +from lbrynet.cryptstream import CryptBlob +from lbrynet.core import HashBlob +from lbrynet.core import log_support + + +log = logging.getLogger('decrypt_blob') + + +def main(): + conf.initialize_settings() + parser = argparse.ArgumentParser() + parser.add_argument('blob_file') + parser.add_argument('hex_key') + parser.add_argument('hex_iv') + parser.add_argument('output') + args = parser.parse_args() + log_support.configure_console() + + d = run(args) + reactor.run() + + +@defer.inlineCallbacks +def run(args): + try: + yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output) + except Exception: + log.exception('Failed to decrypt blob') + finally: + reactor.callLater(0, reactor.stop) + + +@defer.inlineCallbacks +def decrypt_blob(blob_file, key, iv, output): + filename = os.path.abspath(blob_file) + length = os.path.getsize(filename) + directory, blob_hash = os.path.split(filename) + blob = HashBlob.BlobFile(directory, blob_hash, True, length) + decryptor = CryptBlob.StreamBlobDecryptor( + blob, binascii.unhexlify(key), binascii.unhexlify(iv), length) + with open(output, 'w') as f: + yield decryptor.decrypt(f.write) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/encrypt_blob.py b/scripts/encrypt_blob.py new file mode 100644 index 000000000..3408f9023 --- /dev/null +++ b/scripts/encrypt_blob.py @@ -0,0 +1,71 @@ +"""Encrypt a single file using the given key and iv""" +import argparse +import binascii +import logging +import os +import StringIO +import sys + +from twisted.internet import defer +from twisted.internet import reactor + +from lbrynet import conf +from lbrynet.cryptstream import CryptBlob +from lbrynet.core import HashBlob +from lbrynet.core import log_support +from lbrynet.core import cryptoutils + + +log = logging.getLogger('decrypt_blob') + + +def main(): + conf.initialize_settings() + parser = argparse.ArgumentParser() + parser.add_argument('filename') + parser.add_argument('hex_key') + parser.add_argument('hex_iv') + args = parser.parse_args() + log_support.configure_console(level='DEBUG') + + d = run(args) + reactor.run() + + +@defer.inlineCallbacks +def run(args): + try: + yield encrypt_blob(args.filename, args.hex_key, args.hex_iv) + except Exception: + log.exception('Failed to encrypt blob') + finally: + reactor.callLater(0, reactor.stop) + + +def encrypt_blob(filename, key, iv): + blob = Blob() + blob_maker = CryptBlob.CryptStreamBlobMaker( + binascii.unhexlify(key), binascii.unhexlify(iv), 0, blob) + with open(filename) as fin: + blob_maker.write(fin.read()) + blob_maker.close() + + +class Blob(object): + def __init__(self): + self.data = StringIO.StringIO() + + def write(self, data): + self.data.write(data) + + def close(self): + hashsum = cryptoutils.get_lbry_hash_obj() + buffer = self.data.getvalue() + hashsum.update(buffer) + with open(hashsum.hexdigest(), 'w') as fout: + fout.write(buffer) + return defer.succeed(True) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/query_available_blobs.py b/scripts/query_available_blobs.py index a9e321875..61445f4ba 100644 --- a/scripts/query_available_blobs.py +++ b/scripts/query_available_blobs.py @@ -22,7 +22,6 @@ from lbrynet import conf from lbrynet.core import Error from lbrynet.core import Wallet from lbrynet.core import BlobAvailability -from lbrynet.core import BlobManager from lbrynet.core import HashAnnouncer from lbrynet.core import PeerManager from lbrynet.core import Session diff --git a/scripts/reseed_file.py b/scripts/reseed_file.py new file mode 100644 index 000000000..108ff2f00 --- /dev/null +++ b/scripts/reseed_file.py @@ -0,0 +1,94 @@ +"""Reseed a file. + +Given a file and a matching sd_blob, +re-chunk and encrypt the file, adding +the new blobs to the manager. +""" +import argparse +import binascii +import logging +import json +import os +import sys + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.protocols import basic + +from lbrynet import conf +from lbrynet.core import BlobManager +from lbrynet.core import HashAnnouncer +from lbrynet.core import log_support +from lbrynet.cryptstream import CryptStreamCreator + + +log = logging.getLogger('reseed_file') + + +def main(): + conf.initialize_settings() + parser = argparse.ArgumentParser() + parser.add_argument('input_file') + parser.add_argument('sd_blob', help='a json file containing a key and the IVs') + args = parser.parse_args() + log_support.configure_console() + + run(args) + reactor.run() + + +@defer.inlineCallbacks +def run(args): + try: + yield reseed_file(args.input_file, args.sd_blob) + except Exception as e: + log.exception('Failed to reseed') + finally: + reactor.stop() + + +@defer.inlineCallbacks +def reseed_file(input_file, sd_blob): + sd_blob = SdBlob.new_instance(sd_blob) + db_dir = conf.settings['data_dir'] + blobfile_dir = os.path.join(db_dir, "blobfiles") + announcer = HashAnnouncer.DummyHashAnnouncer() + blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir) + yield blob_manager.setup() + creator = CryptStreamCreator.CryptStreamCreator( + blob_manager, None, sd_blob.key(), sd_blob.iv_generator()) + file_sender = basic.FileSender() + with open(input_file) as f: + yield file_sender.beginFileTransfer(f, creator) + yield creator.stop() + for blob_info in sd_blob.blob_infos(): + if 'blob_hash' not in blob_info: + # the last blob is always empty and without a hash + continue + blob = yield blob_manager.get_blob(blob_info['blob_hash'], True) + if not blob.verified: + print "Blob {} is not verified".format(blob) + + +class SdBlob(object): + def __init__(self, contents): + self.contents = contents + + def key(self): + return binascii.unhexlify(self.contents['key']) + + def iv_generator(self): + for blob_info in self.blob_infos(): + yield binascii.unhexlify(blob_info['iv']) + + def blob_infos(self): + return self.contents['blobs'] + + @classmethod + def new_instance(cls, filename): + with open(filename) as f: + return cls(json.load(f)) + + +if __name__ == '__main__': + sys.exit(main())