From 90d3a0ef9c54c501cedf4f8351844512a17975a6 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 20 Jan 2017 14:50:10 -0600 Subject: [PATCH 1/4] Add script to reseed a file --- lbrynet/core/HashBlob.py | 1 + lbrynet/core/StreamCreator.py | 2 + .../lbryfilemanager/EncryptedFileCreator.py | 10 ++ .../lbryfilemanager/EncryptedFileManager.py | 1 + scripts/decrypt_blob.py | 57 +++++++++++ scripts/encrypt_blob.py | 71 ++++++++++++++ scripts/query_available_blobs.py | 1 - scripts/reseed_file.py | 94 +++++++++++++++++++ 8 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 scripts/decrypt_blob.py create mode 100644 scripts/encrypt_blob.py create mode 100644 scripts/reseed_file.py diff --git a/lbrynet/core/HashBlob.py b/lbrynet/core/HashBlob.py index 929d5637e..f09d0709e 100644 --- a/lbrynet/core/HashBlob.py +++ b/lbrynet/core/HashBlob.py @@ -243,6 +243,7 @@ class BlobFile(HashBlob): self.readers += 1 return file_handle except IOError: + log.exception('Failed to open %s', self.file_path) self.close_read_handle(file_handle) return None diff --git a/lbrynet/core/StreamCreator.py b/lbrynet/core/StreamCreator.py index cc579865d..4aa0ae542 100644 --- a/lbrynet/core/StreamCreator.py +++ b/lbrynet/core/StreamCreator.py @@ -64,6 +64,8 @@ class StreamCreator(object): def _finished(self): pass + # TODO: move the stream creation process to its own thread and + # remove the reactor from this process. def write(self, data): from twisted.internet import reactor self._write(data) diff --git a/lbrynet/lbryfilemanager/EncryptedFileCreator.py b/lbrynet/lbryfilemanager/EncryptedFileCreator.py index 43369dbe5..69ae82fd0 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileCreator.py +++ b/lbrynet/lbryfilemanager/EncryptedFileCreator.py @@ -77,6 +77,12 @@ class EncryptedFileStreamCreator(CryptStreamCreator): return d +# TODO: this should be run its own thread. Encrypting a large file can +# be very cpu intensive and there is no need to run that on the +# main reactor thread. The FileSender mechanism that is used is +# great when sending over the network, but this is all local so +# we can simply read the file from the disk without needing to +# involve reactor. def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None, iv_generator=None, suggested_file_name=None): """Turn a plain file into an LBRY File. @@ -146,6 +152,10 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non suggested_file_name) def start_stream(): + # TODO: Using FileSender isn't necessary, we can just read + # straight from the disk. The stream creation process + # should be in its own thread anyway so we don't need to + # worry about interacting with the twisted reactor file_sender = FileSender() d = file_sender.beginFileTransfer(file_handle, lbry_file_creator) d.addCallback(lambda _: stop_file(lbry_file_creator)) diff --git a/lbrynet/lbryfilemanager/EncryptedFileManager.py b/lbrynet/lbryfilemanager/EncryptedFileManager.py index b4ad6b7c4..7791678ab 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileManager.py +++ b/lbrynet/lbryfilemanager/EncryptedFileManager.py @@ -41,6 +41,7 @@ class EncryptedFileManager(object): def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None): self.session = session self.stream_info_manager = stream_info_manager + # TODO: why is sd_identifier part of the file manager? self.sd_identifier = sd_identifier self.lbry_files = [] self.sql_db = None diff --git a/scripts/decrypt_blob.py b/scripts/decrypt_blob.py new file mode 100644 index 000000000..bc905bf2e --- /dev/null +++ b/scripts/decrypt_blob.py @@ -0,0 +1,57 @@ +"""Decrypt a single blob""" +import argparse +import binascii +import logging +import os +import sys + +from twisted.internet import defer +from twisted.internet import reactor + +from lbrynet import conf +from lbrynet.cryptstream import CryptBlob +from lbrynet.core import HashBlob +from lbrynet.core import log_support + + +log = logging.getLogger('decrypt_blob') + + +def main(): + conf.initialize_settings() + parser = argparse.ArgumentParser() + parser.add_argument('blob_file') + parser.add_argument('hex_key') + parser.add_argument('hex_iv') + parser.add_argument('output') + args = parser.parse_args() + log_support.configure_console() + + d = run(args) + reactor.run() + + +@defer.inlineCallbacks +def run(args): + try: + yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output) + except Exception: + log.exception('Failed to decrypt blob') + finally: + reactor.callLater(0, reactor.stop) + + +@defer.inlineCallbacks +def decrypt_blob(blob_file, key, iv, output): + filename = os.path.abspath(blob_file) + length = os.path.getsize(filename) + directory, blob_hash = os.path.split(filename) + blob = HashBlob.BlobFile(directory, blob_hash, True, length) + decryptor = CryptBlob.StreamBlobDecryptor( + blob, binascii.unhexlify(key), binascii.unhexlify(iv), length) + with open(output, 'w') as f: + yield decryptor.decrypt(f.write) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/encrypt_blob.py b/scripts/encrypt_blob.py new file mode 100644 index 000000000..3408f9023 --- /dev/null +++ b/scripts/encrypt_blob.py @@ -0,0 +1,71 @@ +"""Encrypt a single file using the given key and iv""" +import argparse +import binascii +import logging +import os +import StringIO +import sys + +from twisted.internet import defer +from twisted.internet import reactor + +from lbrynet import conf +from lbrynet.cryptstream import CryptBlob +from lbrynet.core import HashBlob +from lbrynet.core import log_support +from lbrynet.core import cryptoutils + + +log = logging.getLogger('decrypt_blob') + + +def main(): + conf.initialize_settings() + parser = argparse.ArgumentParser() + parser.add_argument('filename') + parser.add_argument('hex_key') + parser.add_argument('hex_iv') + args = parser.parse_args() + log_support.configure_console(level='DEBUG') + + d = run(args) + reactor.run() + + +@defer.inlineCallbacks +def run(args): + try: + yield encrypt_blob(args.filename, args.hex_key, args.hex_iv) + except Exception: + log.exception('Failed to encrypt blob') + finally: + reactor.callLater(0, reactor.stop) + + +def encrypt_blob(filename, key, iv): + blob = Blob() + blob_maker = CryptBlob.CryptStreamBlobMaker( + binascii.unhexlify(key), binascii.unhexlify(iv), 0, blob) + with open(filename) as fin: + blob_maker.write(fin.read()) + blob_maker.close() + + +class Blob(object): + def __init__(self): + self.data = StringIO.StringIO() + + def write(self, data): + self.data.write(data) + + def close(self): + hashsum = cryptoutils.get_lbry_hash_obj() + buffer = self.data.getvalue() + hashsum.update(buffer) + with open(hashsum.hexdigest(), 'w') as fout: + fout.write(buffer) + return defer.succeed(True) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/query_available_blobs.py b/scripts/query_available_blobs.py index a9e321875..61445f4ba 100644 --- a/scripts/query_available_blobs.py +++ b/scripts/query_available_blobs.py @@ -22,7 +22,6 @@ from lbrynet import conf from lbrynet.core import Error from lbrynet.core import Wallet from lbrynet.core import BlobAvailability -from lbrynet.core import BlobManager from lbrynet.core import HashAnnouncer from lbrynet.core import PeerManager from lbrynet.core import Session diff --git a/scripts/reseed_file.py b/scripts/reseed_file.py new file mode 100644 index 000000000..108ff2f00 --- /dev/null +++ b/scripts/reseed_file.py @@ -0,0 +1,94 @@ +"""Reseed a file. + +Given a file and a matching sd_blob, +re-chunk and encrypt the file, adding +the new blobs to the manager. +""" +import argparse +import binascii +import logging +import json +import os +import sys + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.protocols import basic + +from lbrynet import conf +from lbrynet.core import BlobManager +from lbrynet.core import HashAnnouncer +from lbrynet.core import log_support +from lbrynet.cryptstream import CryptStreamCreator + + +log = logging.getLogger('reseed_file') + + +def main(): + conf.initialize_settings() + parser = argparse.ArgumentParser() + parser.add_argument('input_file') + parser.add_argument('sd_blob', help='a json file containing a key and the IVs') + args = parser.parse_args() + log_support.configure_console() + + run(args) + reactor.run() + + +@defer.inlineCallbacks +def run(args): + try: + yield reseed_file(args.input_file, args.sd_blob) + except Exception as e: + log.exception('Failed to reseed') + finally: + reactor.stop() + + +@defer.inlineCallbacks +def reseed_file(input_file, sd_blob): + sd_blob = SdBlob.new_instance(sd_blob) + db_dir = conf.settings['data_dir'] + blobfile_dir = os.path.join(db_dir, "blobfiles") + announcer = HashAnnouncer.DummyHashAnnouncer() + blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir) + yield blob_manager.setup() + creator = CryptStreamCreator.CryptStreamCreator( + blob_manager, None, sd_blob.key(), sd_blob.iv_generator()) + file_sender = basic.FileSender() + with open(input_file) as f: + yield file_sender.beginFileTransfer(f, creator) + yield creator.stop() + for blob_info in sd_blob.blob_infos(): + if 'blob_hash' not in blob_info: + # the last blob is always empty and without a hash + continue + blob = yield blob_manager.get_blob(blob_info['blob_hash'], True) + if not blob.verified: + print "Blob {} is not verified".format(blob) + + +class SdBlob(object): + def __init__(self, contents): + self.contents = contents + + def key(self): + return binascii.unhexlify(self.contents['key']) + + def iv_generator(self): + for blob_info in self.blob_infos(): + yield binascii.unhexlify(blob_info['iv']) + + def blob_infos(self): + return self.contents['blobs'] + + @classmethod + def new_instance(cls, filename): + with open(filename) as f: + return cls(json.load(f)) + + +if __name__ == '__main__': + sys.exit(main()) From c92066344480591f3a6049bd6e6937407e4c070b Mon Sep 17 00:00:00 2001 From: jobevers Date: Tue, 21 Feb 2017 11:36:44 -0600 Subject: [PATCH 2/4] =?UTF-8?q?Bump=20version:=200.8.6=20=E2=86=92=200.8.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- CHANGELOG.md | 1 + lbrynet/__init__.py | 2 +- packaging/ubuntu/lbry.desktop | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0693a5f94..f86606b00 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.6 +current_version = 0.8.7 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)((?P[a-z]+)(?P\d+))? diff --git a/CHANGELOG.md b/CHANGELOG.md index 65fb03666..a251076ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ can and probably will change functionality and break backwards compatability at anytime. ## [Unreleased] +## [0.8.7] - 2017-02-21 \#\# [0.8.6] - 2017-02-19 \#\# [0.8.6rc0] - 2017-02-19 ### Changed diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 67fa5618e..6eb26a994 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,6 +1,6 @@ import logging -__version__ = "0.8.6" +__version__ = "0.8.7" version = tuple(__version__.split('.')) logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index 5431b0ed5..3d46d20d1 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.8.6 +Version=0.8.7 Name=LBRY Comment=The world's first user-owned content marketplace Icon=lbry From 62ab4c769da3c6580e3c976fcfaac513c5f71614 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Tue, 21 Feb 2017 12:51:13 -0500 Subject: [PATCH 3/4] move blocks_behind into blockchain_status --- lbrynet/lbrynet_daemon/Daemon.py | 43 ++++++++++++++--------------- lbrynet/lbrynet_daemon/DaemonCLI.py | 4 +-- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index c65465005..62c1d282a 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -1107,7 +1107,7 @@ class Daemon(AuthJSONRPCServer): ############################################################################ @defer.inlineCallbacks - def jsonrpc_status(self, session_status=False, blockchain_status=False): + def jsonrpc_status(self, session_status=False): """ Return daemon status @@ -1118,6 +1118,10 @@ class Daemon(AuthJSONRPCServer): daemon status """ has_wallet = self.session and self.session.wallet + local_height = self.session.wallet.network.get_local_height() if has_wallet else 0 + remote_height = self.session.wallet.network.get_server_height() if has_wallet else 0 + best_hash = (yield self.session.wallet.get_best_blockhash()) if has_wallet else None + response = { 'lbry_id': base58.b58encode(self.lbryid)[:SHORT_ID_LEN], 'installation_id': conf.settings.get_installation_id()[:SHORT_ID_LEN], @@ -1135,11 +1139,12 @@ class Daemon(AuthJSONRPCServer): else '' ), }, - 'blocks_behind': ( - self.session.wallet.blocks_behind - if has_wallet and self.wallet_type == LBRYUM_WALLET - else 'unknown' - ), + 'blocks_behind': remote_height - local_height, # deprecated. remove from UI, then here + 'blockchain_status': { + 'blocks': local_height, + 'blocks_behind': remote_height - local_height, + 'best_blockhash': best_hash, + } } if session_status: blobs = yield self.session.blob_manager.get_all_verified_blobs() @@ -1147,22 +1152,14 @@ class Daemon(AuthJSONRPCServer): 'managed_blobs': len(blobs), 'managed_streams': len(self.lbry_file_manager.lbry_files), } - if blockchain_status and has_wallet: - # calculate blocks_behind more accurately - local_height = self.session.wallet.network.get_local_height() - remote_height = self.session.wallet.network.get_server_height() - response['blocks_behind'] = remote_height - local_height - response['local_height'] = local_height - response['remote_height'] = remote_height - best_hash = yield self.session.wallet.get_best_blockhash() - response['blockchain_status'] = {'best_blockhash': best_hash} + defer.returnValue(response) def jsonrpc_get_best_blockhash(self): """ DEPRECATED. Use `status blockchain_status=True` instead """ - d = self.jsonrpc_status(blockchain_status=True) + d = self.jsonrpc_status() d.addCallback(lambda x: self._render_response( x['blockchain_status']['best_blockhash'])) return d @@ -1191,9 +1188,11 @@ class Daemon(AuthJSONRPCServer): elif status['startup_status']['code'] == LOADING_WALLET_CODE: message = "Catching up with the blockchain." progress = 0 - if status['blocks_behind'] > 0: - message += ' ' + str(status['blocks_behind']) + " blocks behind." - progress = status['blocks_behind'] + if status['blockchain_status']['blocks_behind'] > 0: + message += ( + ' ' + str(status['blockchain_status']['blocks_behind']) + " blocks behind." + ) + progress = status['blockchain_status']['blocks_behind'] return { 'message': message, @@ -1212,7 +1211,7 @@ class Daemon(AuthJSONRPCServer): """ DEPRECATED. Use `status` instead """ - d = self.jsonrpc_status(blockchain_status=True) + d = self.jsonrpc_status() d.addCallback(lambda x: self._render_response(x['is_first_run'])) return d @@ -1233,8 +1232,8 @@ class Daemon(AuthJSONRPCServer): """ DEPRECATED. Use `status` instead """ - d = self.jsonrpc_status(blockchain_status=True) - d.addCallback(lambda x: self._render_response(x['blocks_behind'])) + d = self.jsonrpc_status() + d.addCallback(lambda x: self._render_response(x['blockchain_status']['blocks_behind'])) return d def jsonrpc_version(self): diff --git a/lbrynet/lbrynet_daemon/DaemonCLI.py b/lbrynet/lbrynet_daemon/DaemonCLI.py index 98f0a69ab..96e4035b0 100644 --- a/lbrynet/lbrynet_daemon/DaemonCLI.py +++ b/lbrynet/lbrynet_daemon/DaemonCLI.py @@ -35,9 +35,9 @@ def main(): if message: if ( status['startup_status']['code'] == LOADING_WALLET_CODE - and status['blocks_behind'] > 0 + and status['blockchain_status']['blocks_behind'] > 0 ): - message += '. Blocks left: ' + str(status['blocks_behind']) + message += '. Blocks left: ' + str(status['blockchain_status']['blocks_behind']) print " Status: " + message return 1 From b6086a7d6dd792def6ebfc74927ed3f3069a1e54 Mon Sep 17 00:00:00 2001 From: jobevers Date: Tue, 21 Feb 2017 11:49:51 -0600 Subject: [PATCH 4/4] cleanup changelog --- CHANGELOG.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a251076ba..492668f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,24 +8,26 @@ can and probably will change functionality and break backwards compatability at anytime. ## [Unreleased] + ## [0.8.7] - 2017-02-21 -\#\# [0.8.6] - 2017-02-19 -\#\# [0.8.6rc0] - 2017-02-19 + +## [0.8.6] - 2017-02-19 + +## [0.8.6rc0] - 2017-02-19 ### Changed * Add `file_get` by stream hash * Add utils.call_later to replace reactor.callLater -### + +### Fixed * Fix unhandled error in `get` * Fix sd blob timeout handling in `get_availability`, return 0.0 - - + ## [0.8.5] - 2017-02-18 ## [0.8.5rc0] - 2017-02-18 ### Fixed * Fix result expected by ui from file_get for missing files - ## [0.8.4] - 2017-02-17 ## [0.8.4rc0] - 2017-02-17 @@ -35,19 +37,18 @@ at anytime. ### Fixed * add misssing traceback to logging - ## [0.8.3] - 2017-02-15 ### Fixed * Get lbry files with pending claims * Add better logging to help track down [#478](https://github.com/lbryio/lbry/issues/478) * Catch UnknownNameErrors when resolving a name. [#479](https://github.com/lbryio/lbry/issues/479) + ### Changed * Add blob_get, descriptor_get, and blob_delete * Add filter keyword args to blob_list * Refactor get_availability * Add optional peer search timeout, add peer_search_timeout setting - ## [0.8.3rc3] - 2017-02-14 ## [0.8.3rc2] - 2017-02-13