From 90d3a0ef9c54c501cedf4f8351844512a17975a6 Mon Sep 17 00:00:00 2001
From: Job Evers-Meltzer <jobevers@users.noreply.github.com>
Date: Fri, 20 Jan 2017 14:50:10 -0600
Subject: [PATCH] Add script to reseed a file
---
lbrynet/core/HashBlob.py | 1 +
lbrynet/core/StreamCreator.py | 2 +
.../lbryfilemanager/EncryptedFileCreator.py | 10 ++
.../lbryfilemanager/EncryptedFileManager.py | 1 +
scripts/decrypt_blob.py | 57 +++++++++++
scripts/encrypt_blob.py | 71 ++++++++++++++
scripts/query_available_blobs.py | 1 -
scripts/reseed_file.py | 94 +++++++++++++++++++
8 files changed, 236 insertions(+), 1 deletion(-)
create mode 100644 scripts/decrypt_blob.py
create mode 100644 scripts/encrypt_blob.py
create mode 100644 scripts/reseed_file.py
diff --git a/lbrynet/core/HashBlob.py b/lbrynet/core/HashBlob.py
index 929d5637e..f09d0709e 100644
--- a/lbrynet/core/HashBlob.py
+++ b/lbrynet/core/HashBlob.py
@@ -243,6 +243,7 @@ class BlobFile(HashBlob):
self.readers += 1
return file_handle
except IOError:
+ log.exception('Failed to open %s', self.file_path)
self.close_read_handle(file_handle)
return None
diff --git a/lbrynet/core/StreamCreator.py b/lbrynet/core/StreamCreator.py
index cc579865d..4aa0ae542 100644
--- a/lbrynet/core/StreamCreator.py
+++ b/lbrynet/core/StreamCreator.py
@@ -64,6 +64,8 @@ class StreamCreator(object):
def _finished(self):
pass
+ # TODO: move the stream creation process to its own thread and
+ # remove the reactor from this process.
def write(self, data):
from twisted.internet import reactor
self._write(data)
diff --git a/lbrynet/lbryfilemanager/EncryptedFileCreator.py b/lbrynet/lbryfilemanager/EncryptedFileCreator.py
index 43369dbe5..69ae82fd0 100644
--- a/lbrynet/lbryfilemanager/EncryptedFileCreator.py
+++ b/lbrynet/lbryfilemanager/EncryptedFileCreator.py
@@ -77,6 +77,12 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
return d
+# TODO: this should be run its own thread. Encrypting a large file can
+# be very cpu intensive and there is no need to run that on the
+# main reactor thread. The FileSender mechanism that is used is
+# great when sending over the network, but this is all local so
+# we can simply read the file from the disk without needing to
+# involve reactor.
def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None,
iv_generator=None, suggested_file_name=None):
"""Turn a plain file into an LBRY File.
@@ -146,6 +152,10 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
suggested_file_name)
def start_stream():
+ # TODO: Using FileSender isn't necessary, we can just read
+ # straight from the disk. The stream creation process
+ # should be in its own thread anyway so we don't need to
+ # worry about interacting with the twisted reactor
file_sender = FileSender()
d = file_sender.beginFileTransfer(file_handle, lbry_file_creator)
d.addCallback(lambda _: stop_file(lbry_file_creator))
diff --git a/lbrynet/lbryfilemanager/EncryptedFileManager.py b/lbrynet/lbryfilemanager/EncryptedFileManager.py
index b4ad6b7c4..7791678ab 100644
--- a/lbrynet/lbryfilemanager/EncryptedFileManager.py
+++ b/lbrynet/lbryfilemanager/EncryptedFileManager.py
@@ -41,6 +41,7 @@ class EncryptedFileManager(object):
def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
self.session = session
self.stream_info_manager = stream_info_manager
+ # TODO: why is sd_identifier part of the file manager?
self.sd_identifier = sd_identifier
self.lbry_files = []
self.sql_db = None
diff --git a/scripts/decrypt_blob.py b/scripts/decrypt_blob.py
new file mode 100644
index 000000000..bc905bf2e
--- /dev/null
+++ b/scripts/decrypt_blob.py
@@ -0,0 +1,57 @@
+"""Decrypt a single blob"""
+import argparse
+import binascii
+import logging
+import os
+import sys
+
+from twisted.internet import defer
+from twisted.internet import reactor
+
+from lbrynet import conf
+from lbrynet.cryptstream import CryptBlob
+from lbrynet.core import HashBlob
+from lbrynet.core import log_support
+
+
+log = logging.getLogger('decrypt_blob')
+
+
+def main():
+ conf.initialize_settings()
+ parser = argparse.ArgumentParser()
+ parser.add_argument('blob_file')
+ parser.add_argument('hex_key')
+ parser.add_argument('hex_iv')
+ parser.add_argument('output')
+ args = parser.parse_args()
+ log_support.configure_console()
+
+ d = run(args)
+ reactor.run()
+
+
+@defer.inlineCallbacks
+def run(args):
+ try:
+ yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output)
+ except Exception:
+ log.exception('Failed to decrypt blob')
+ finally:
+ reactor.callLater(0, reactor.stop)
+
+
+@defer.inlineCallbacks
+def decrypt_blob(blob_file, key, iv, output):
+ filename = os.path.abspath(blob_file)
+ length = os.path.getsize(filename)
+ directory, blob_hash = os.path.split(filename)
+ blob = HashBlob.BlobFile(directory, blob_hash, True, length)
+ decryptor = CryptBlob.StreamBlobDecryptor(
+ blob, binascii.unhexlify(key), binascii.unhexlify(iv), length)
+ with open(output, 'w') as f:
+ yield decryptor.decrypt(f.write)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/scripts/encrypt_blob.py b/scripts/encrypt_blob.py
new file mode 100644
index 000000000..3408f9023
--- /dev/null
+++ b/scripts/encrypt_blob.py
@@ -0,0 +1,71 @@
+"""Encrypt a single file using the given key and iv"""
+import argparse
+import binascii
+import logging
+import os
+import StringIO
+import sys
+
+from twisted.internet import defer
+from twisted.internet import reactor
+
+from lbrynet import conf
+from lbrynet.cryptstream import CryptBlob
+from lbrynet.core import HashBlob
+from lbrynet.core import log_support
+from lbrynet.core import cryptoutils
+
+
+log = logging.getLogger('decrypt_blob')
+
+
+def main():
+ conf.initialize_settings()
+ parser = argparse.ArgumentParser()
+ parser.add_argument('filename')
+ parser.add_argument('hex_key')
+ parser.add_argument('hex_iv')
+ args = parser.parse_args()
+ log_support.configure_console(level='DEBUG')
+
+ d = run(args)
+ reactor.run()
+
+
+@defer.inlineCallbacks
+def run(args):
+ try:
+ yield encrypt_blob(args.filename, args.hex_key, args.hex_iv)
+ except Exception:
+ log.exception('Failed to encrypt blob')
+ finally:
+ reactor.callLater(0, reactor.stop)
+
+
+def encrypt_blob(filename, key, iv):
+ blob = Blob()
+ blob_maker = CryptBlob.CryptStreamBlobMaker(
+ binascii.unhexlify(key), binascii.unhexlify(iv), 0, blob)
+ with open(filename) as fin:
+ blob_maker.write(fin.read())
+ blob_maker.close()
+
+
+class Blob(object):
+ def __init__(self):
+ self.data = StringIO.StringIO()
+
+ def write(self, data):
+ self.data.write(data)
+
+ def close(self):
+ hashsum = cryptoutils.get_lbry_hash_obj()
+ buffer = self.data.getvalue()
+ hashsum.update(buffer)
+ with open(hashsum.hexdigest(), 'w') as fout:
+ fout.write(buffer)
+ return defer.succeed(True)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/scripts/query_available_blobs.py b/scripts/query_available_blobs.py
index a9e321875..61445f4ba 100644
--- a/scripts/query_available_blobs.py
+++ b/scripts/query_available_blobs.py
@@ -22,7 +22,6 @@ from lbrynet import conf
from lbrynet.core import Error
from lbrynet.core import Wallet
from lbrynet.core import BlobAvailability
-from lbrynet.core import BlobManager
from lbrynet.core import HashAnnouncer
from lbrynet.core import PeerManager
from lbrynet.core import Session
diff --git a/scripts/reseed_file.py b/scripts/reseed_file.py
new file mode 100644
index 000000000..108ff2f00
--- /dev/null
+++ b/scripts/reseed_file.py
@@ -0,0 +1,94 @@
+"""Reseed a file.
+
+Given a file and a matching sd_blob,
+re-chunk and encrypt the file, adding
+the new blobs to the manager.
+"""
+import argparse
+import binascii
+import logging
+import json
+import os
+import sys
+
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.protocols import basic
+
+from lbrynet import conf
+from lbrynet.core import BlobManager
+from lbrynet.core import HashAnnouncer
+from lbrynet.core import log_support
+from lbrynet.cryptstream import CryptStreamCreator
+
+
+log = logging.getLogger('reseed_file')
+
+
+def main():
+ conf.initialize_settings()
+ parser = argparse.ArgumentParser()
+ parser.add_argument('input_file')
+ parser.add_argument('sd_blob', help='a json file containing a key and the IVs')
+ args = parser.parse_args()
+ log_support.configure_console()
+
+ run(args)
+ reactor.run()
+
+
+@defer.inlineCallbacks
+def run(args):
+ try:
+ yield reseed_file(args.input_file, args.sd_blob)
+ except Exception as e:
+ log.exception('Failed to reseed')
+ finally:
+ reactor.stop()
+
+
+@defer.inlineCallbacks
+def reseed_file(input_file, sd_blob):
+ sd_blob = SdBlob.new_instance(sd_blob)
+ db_dir = conf.settings['data_dir']
+ blobfile_dir = os.path.join(db_dir, "blobfiles")
+ announcer = HashAnnouncer.DummyHashAnnouncer()
+ blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir)
+ yield blob_manager.setup()
+ creator = CryptStreamCreator.CryptStreamCreator(
+ blob_manager, None, sd_blob.key(), sd_blob.iv_generator())
+ file_sender = basic.FileSender()
+ with open(input_file) as f:
+ yield file_sender.beginFileTransfer(f, creator)
+ yield creator.stop()
+ for blob_info in sd_blob.blob_infos():
+ if 'blob_hash' not in blob_info:
+ # the last blob is always empty and without a hash
+ continue
+ blob = yield blob_manager.get_blob(blob_info['blob_hash'], True)
+ if not blob.verified:
+ print "Blob {} is not verified".format(blob)
+
+
+class SdBlob(object):
+ def __init__(self, contents):
+ self.contents = contents
+
+ def key(self):
+ return binascii.unhexlify(self.contents['key'])
+
+ def iv_generator(self):
+ for blob_info in self.blob_infos():
+ yield binascii.unhexlify(blob_info['iv'])
+
+ def blob_infos(self):
+ return self.contents['blobs']
+
+ @classmethod
+ def new_instance(cls, filename):
+ with open(filename) as f:
+ return cls(json.load(f))
+
+
+if __name__ == '__main__':
+ sys.exit(main())