From f74f075b4eb2d285cec5efd6f0242f4b4bcc8e4c Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sun, 10 Jul 2016 17:47:36 -0500 Subject: [PATCH 01/36] add tests for BlobRequestHandler and run on travis --- .../install_dependencies_and_run_tests.sh | 4 +- tests/__init__.py | 0 tests/lbrynet/__init__.py | 0 tests/lbrynet/core/__init__.py | 0 tests/lbrynet/core/server/__init__.py | 0 .../core/server/test_BlobRequestHandler.py | 104 ++++++++++++++++++ 6 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/lbrynet/__init__.py create mode 100644 tests/lbrynet/core/__init__.py create mode 100644 tests/lbrynet/core/server/__init__.py create mode 100644 tests/lbrynet/core/server/test_BlobRequestHandler.py diff --git a/packaging/travis/install_dependencies_and_run_tests.sh b/packaging/travis/install_dependencies_and_run_tests.sh index f55ab609b..6014087e1 100755 --- a/packaging/travis/install_dependencies_and_run_tests.sh +++ b/packaging/travis/install_dependencies_and_run_tests.sh @@ -38,8 +38,8 @@ rm get-pip.py pip install -r requirements.txt -pip install nose coverage coveralls pylint -nosetests --with-coverage --cover-package=lbrynet -v -I functional_tests.py tests/ +pip install mock pylint +trial tests # TODO: submit coverage report to coveralls # TODO: as code quality improves, make pylint be more strict diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/__init__.py b/tests/lbrynet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/core/__init__.py b/tests/lbrynet/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/core/server/__init__.py b/tests/lbrynet/core/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/core/server/test_BlobRequestHandler.py b/tests/lbrynet/core/server/test_BlobRequestHandler.py new file mode 100644 index 000000000..4cd04259a --- /dev/null +++ b/tests/lbrynet/core/server/test_BlobRequestHandler.py @@ -0,0 +1,104 @@ +import mock +from twisted.internet import defer +from twisted.trial import unittest + +from lbrynet.core.server import BlobRequestHandler + + +class TestBlobRequestHandlerQueries(unittest.TestCase): + def setUp(self): + self.blob_manager = mock.Mock() + self.payment_rate_manager = mock.Mock() + self.handler = BlobRequestHandler.BlobRequestHandler( + self.blob_manager, None, self.payment_rate_manager) + + def test_empty_response_when_empty_query(self): + self.assertEqual({}, self.successResultOf(self.handler.handle_queries({}))) + + def test_error_set_when_rate_is_missing(self): + query = {'requested_blob': 'blob'} + deferred = self.handler.handle_queries(query) + response = {'incoming_blob': {'error': 'RATE_UNSET'}} + self.assertEqual(response, self.successResultOf(deferred)) + + def test_error_set_when_rate_too_low(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = False + query = { + 'blob_data_payment_rate': 'way_too_low', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_TOO_LOW', + 'incoming_blob': {'error': 'RATE_UNSET'} + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_response_when_rate_too_low(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = False + query = { + 'blob_data_payment_rate': 'way_too_low', + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_TOO_LOW', + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_blob_unavailable_when_blob_not_validated(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = True + blob = mock.Mock() + blob.is_validated.return_value = False + self.blob_manager.get_blob.return_value = defer.succeed(blob) + query = { + 'blob_data_payment_rate': 'rate', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_ACCEPTED', + 'incoming_blob': {'error': 'BLOB_UNAVAILABLE'} + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_blob_unavailable_when_blob_cannot_be_opened(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = True + blob = mock.Mock() + blob.is_validated.return_value = True + blob.open_for_reading.return_value = None + self.blob_manager.get_blob.return_value = defer.succeed(blob) + query = { + 'blob_data_payment_rate': 'rate', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_ACCEPTED', + 'incoming_blob': {'error': 'BLOB_UNAVAILABLE'} + } + self.assertEqual(response, self.successResultOf(deferred)) + + def test_blob_details_are_set_when_all_conditions_are_met(self): + self.payment_rate_manager.accept_rate_blob_data.return_value = True + blob = mock.Mock() + blob.is_validated.return_value = True + blob.open_for_reading.return_value = True + blob.blob_hash = 'DEADBEEF' + blob.length = 42 + self.blob_manager.get_blob.return_value = defer.succeed(blob) + query = { + 'blob_data_payment_rate': 'rate', + 'requested_blob': 'blob' + } + deferred = self.handler.handle_queries(query) + response = { + 'blob_data_payment_rate': 'RATE_ACCEPTED', + 'incoming_blob': { + 'blob_hash': 'DEADBEEF', + 'length': 42 + } + } + self.assertEqual(response, self.successResultOf(deferred)) + + + From 53bf1fe4fd7da700051b8c4a75d953361cc99747 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sun, 10 Jul 2016 21:56:00 -0500 Subject: [PATCH 02/36] add send_blob tests --- lbrynet/core/server/BlobRequestHandler.py | 3 +- .../core/server/test_BlobRequestHandler.py | 33 ++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/lbrynet/core/server/BlobRequestHandler.py b/lbrynet/core/server/BlobRequestHandler.py index 2ce3688b4..36f67d797 100644 --- a/lbrynet/core/server/BlobRequestHandler.py +++ b/lbrynet/core/server/BlobRequestHandler.py @@ -140,6 +140,7 @@ class BlobRequestHandler(object): def set_expected_payment(): log.info("Setting expected payment") if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None: + # TODO: explain why 2**20 self.wallet.add_expected_payment(self.peer, self.currently_uploading.length * 1.0 * self.blob_data_payment_rate / 2**20) @@ -156,4 +157,4 @@ class BlobRequestHandler(object): if reason is not None and isinstance(reason, Failure): log.info("Upload has failed. Reason: %s", reason.getErrorMessage()) - return _send_file() \ No newline at end of file + return _send_file() diff --git a/tests/lbrynet/core/server/test_BlobRequestHandler.py b/tests/lbrynet/core/server/test_BlobRequestHandler.py index 4cd04259a..5c55af574 100644 --- a/tests/lbrynet/core/server/test_BlobRequestHandler.py +++ b/tests/lbrynet/core/server/test_BlobRequestHandler.py @@ -1,7 +1,11 @@ +import StringIO + import mock -from twisted.internet import defer +from twisted.internet import defer, protocol +from twisted.test import proto_helpers from twisted.trial import unittest +from lbrynet.core import Peer from lbrynet.core.server import BlobRequestHandler @@ -13,8 +17,9 @@ class TestBlobRequestHandlerQueries(unittest.TestCase): self.blob_manager, None, self.payment_rate_manager) def test_empty_response_when_empty_query(self): - self.assertEqual({}, self.successResultOf(self.handler.handle_queries({}))) - + self.assertEqual( + {}, self.successResultOf(self.handler.handle_queries({}))) + def test_error_set_when_rate_is_missing(self): query = {'requested_blob': 'blob'} deferred = self.handler.handle_queries(query) @@ -99,6 +104,24 @@ class TestBlobRequestHandlerQueries(unittest.TestCase): } } self.assertEqual(response, self.successResultOf(deferred)) - - + +class TestBlobRequestHandlerSender(unittest.TestCase): + def test_nothing_happens_if_not_currently_uploading(self): + handler = BlobRequestHandler.BlobRequestHandler(None, None, None) + handler.currently_uploading = None + deferred = handler.send_blob_if_requested(None) + self.assertEqual(True, self.successResultOf(deferred)) + + def test_file_is_sent_to_consumer(self): + # TODO: also check that the expected payment values are set + consumer = proto_helpers.StringTransport() + test_file = StringIO.StringIO('test') + handler = BlobRequestHandler.BlobRequestHandler(None, None, None) + handler.peer = mock.create_autospec(Peer.Peer) + handler.currently_uploading = mock.Mock() + handler.read_handle = test_file + handler.send_blob_if_requested(consumer) + while consumer.producer: + consumer.producer.resumeProducing() + self.assertEqual(consumer.value(), 'test') From ee71a03f96bbb5241a3da52f08cf2480b1b847b3 Mon Sep 17 00:00:00 2001 From: kimihiro64 Date: Wed, 13 Jul 2016 22:44:02 -0500 Subject: [PATCH 03/36] Revert to 2 params as isinstance cannot support 3+ (#69) * Revert to 2 params as isinstance cannot support 3+ * Use basestring for maximum efficiency --- lbrynet/lbrynet_daemon/LBRYDaemonCLI.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py index 2b34d4301..0b5d0ba0c 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py @@ -35,7 +35,7 @@ def main(): if len(args) > 1: if isinstance(args[1], dict): params = args[1] - elif isinstance(args[1], str, unicode): + elif isinstance(args[1], basestring): params = json.loads(args[1]) else: params = None @@ -56,4 +56,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() From 7ffb3ea04220c593ad1dc7da26c29164458ea9d5 Mon Sep 17 00:00:00 2001 From: Harwinder Date: Thu, 14 Jul 2016 18:41:23 +0200 Subject: [PATCH 04/36] Added git to linux install command Some people dont have git installed so adding it to it solves confusion for people new to linux. --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 23491e60c..8bf9f8917 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -23,7 +23,7 @@ You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio On Ubuntu or Mint you can install the prerequisites and lbrynet by running ``` - sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip + sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git git clone https://github.com/lbryio/lbry.git cd lbry sudo python setup.py install From 1e48f5c8f409f4754497cc8969be11134f78d131 Mon Sep 17 00:00:00 2001 From: Harwinder Date: Thu, 14 Jul 2016 22:25:32 +0200 Subject: [PATCH 05/36] Cant use git if its not installed First install needed packages then execute the task. Cant run the whole code if git isnt installed :) --- RUNNING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RUNNING.md b/RUNNING.md index b9f60acc9..d2f1842b4 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -43,9 +43,9 @@ To stop lbrynet-console, enter the command 'exit'. Note: this process takes upwards of an hour and is not necessary to use lbrynet. ``` +sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler git git clone --depth=1 -b alpha https://github.com/lbryio/lbrycrd.git cd lbrycrd -sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler ./autogen.sh ./configure --with-incompatible-bdb --without-gui From 5aecd02668c68eff15e7cb1640304a6ab3c9595e Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 15 Jul 2016 11:09:20 -0500 Subject: [PATCH 06/36] Refactor jsonrpc_get. Move parameter handling into its own function and better use the `.get()` function for dictionaries. Early return on the failed checks is more readable. The lambda function in the callback was long and hard to read so moved it out. --- .pylintrc | 2 +- lbrynet/lbrynet_daemon/LBRYDaemon.py | 107 +++++++++++++++------------ 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/.pylintrc b/.pylintrc index 5e5aab35c..e49732469 100644 --- a/.pylintrc +++ b/.pylintrc @@ -298,7 +298,7 @@ ignored-classes=twisted.internet,RequestMessage # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters [IMPORTS] diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 8e2c98cc5..c5280bfd2 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -138,6 +138,11 @@ OK_CODE = 200 REMOTE_SERVER = "www.google.com" +class Parameters(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + class LBRYDaemon(jsonrpc.JSONRPC): """ LBRYnet daemon, a jsonrpc interface to lbry functions @@ -1651,65 +1656,53 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d + def _process_get_parameters(self, p): + """Extract info from input parameters and fill in default values for `get` call.""" + # TODO: this process can be abstracted s.t. each method + # can spec what parameters it expects and how to set default values + timeout = p.get('timeout', self.download_timeout) + download_directory = p.get('download_directory', self.download_directory) + file_name = p.get('file_name') + stream_info = p.get('stream_info') + sd_hash = get_sd_hash(stream_info) + wait_for_write = p.get('wait_for_write', True) + name = p.get('name') + return Parameters( + timout=timeout, + download_directory=download_directory, + file_name=file_name, + stream_info=stream_info, + sd_hash=sd_hash, + wait_for_write=wait_for_write, + name=name + ) + def jsonrpc_get(self, p): - """ - Download stream from a LBRY uri + """Download stream from a LBRY uri. Args: 'name': name to download, string 'download_directory': optional, path to directory where file will be saved, string 'file_name': optional, a user specified name for the downloaded file 'stream_info': optional, specified stream info overrides name + 'timout': optional Returns: 'stream_hash': hex string 'path': path of download """ - - if 'timeout' not in p.keys(): - timeout = self.download_timeout - else: - timeout = p['timeout'] - - if 'download_directory' not in p.keys(): - download_directory = self.download_directory - else: - download_directory = p['download_directory'] - - if 'file_name' in p.keys(): - file_name = p['file_name'] - else: - file_name = None - - if 'stream_info' in p.keys(): - stream_info = p['stream_info'] - if 'sources' in stream_info.keys(): - sd_hash = stream_info['sources']['lbry_sd_hash'] - else: - sd_hash = stream_info['stream_hash'] - else: - stream_info = None - - if 'wait_for_write' in p.keys(): - wait_for_write = p['wait_for_write'] - else: - wait_for_write = True - - if 'name' in p.keys(): - name = p['name'] - if p['name'] not in self.waiting_on.keys(): - d = self._download_name(name=name, timeout=timeout, download_directory=download_directory, - stream_info=stream_info, file_name=file_name, wait_for_write=wait_for_write) - d.addCallback(lambda l: {'stream_hash': sd_hash, - 'path': os.path.join(self.download_directory, l.file_name)} - if stream_info else - {'stream_hash': l.sd_hash, - 'path': os.path.join(self.download_directory, l.file_name)}) - d.addCallback(lambda message: self._render_response(message, OK_CODE)) - else: - d = server.failure - else: - d = server.failure - + params = self._process_get_parameters(p) + if not params.name: + return server.failure + if params.name in self.waiting_on: + return server.failure + d = self._download_name(name=params.name, + timeout=params.timeout, + download_directory=params.download_directory, + stream_info=params.stream_info, + file_name=params.file_name, + wait_for_write=params.wait_for_write) + d.addCallback(get_output_callback(params)) + d.addCallback(lambda message: self._render_response(message, OK_CODE)) return d def jsonrpc_stop_lbry_file(self, p): @@ -2261,3 +2254,21 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d + + +def get_sd_hash(stream_info): + if not stream_info: + return None + try: + return stream_info['sources']['lbry_sd_hash'] + except KeyError: + return stream_info.get('stream_hash') + + +def get_output_callback(params): + def callback(l): + return { + 'stream_hash': params.sd_hash if params.stream_info else l.sd_hash, + 'path': os.path.join(params.download_directory, l.file_name) + } + return callback From 563896b126564d6e249cea3486cc193e52db0fb2 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 15 Jul 2016 11:33:38 -0500 Subject: [PATCH 07/36] fix bug in reveal code --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index c5280bfd2..6fa942b22 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -2249,7 +2249,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = threads.deferToThread(subprocess.Popen, ['open', '-R', path]) else: # No easy way to reveal specific files on Linux, so just open the containing directory - d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.dirname(path)]) + d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.path.dirname(path)]) d.addCallback(lambda _: self._render_response(True, OK_CODE)) From a90029ec50bbb94ec4625a4d9b57f367edd1b7b4 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 15 Jul 2016 11:37:04 -0500 Subject: [PATCH 08/36] update doc string --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 6fa942b22..0c2ce0303 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1686,6 +1686,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'file_name': optional, a user specified name for the downloaded file 'stream_info': optional, specified stream info overrides name 'timout': optional + 'wait_for_write': optional, defaults to True Returns: 'stream_hash': hex string 'path': path of download From a15d7ca54311f3248427bcf3a1b4eb42d1fc889a Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 15 Jul 2016 12:42:26 -0500 Subject: [PATCH 09/36] Refactor _download_name Nested functions are the devil, especially ones that use variables from the outer scope. Refactoring _download_name to use a helper class helps make the scoping more explicit and will undoubtably prevent bugs in the future. I think this makes _download_name drastically more readable. Also cleaned up some duplicated code and made download_directory respect the passed in parameter instead of being the default. --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 189 +++++++++++++++------------ 1 file changed, 109 insertions(+), 80 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 0c2ce0303..fad2583bd 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -999,97 +999,37 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None, - file_name=None, stream_info=None, wait_for_write=True): + file_name=None, stream_info=None, wait_for_write=True): """ Add a lbry file to the file manager, start the download, and return the new lbry file. If it already exists in the file manager, return the existing lbry file """ - - if not download_directory: - download_directory = self.download_directory - elif not os.path.isdir(download_directory): - download_directory = self.download_directory - - def _remove_from_wait(r): - del self.waiting_on[name] - return r - - def _setup_stream(stream_info): - if 'sources' in stream_info.keys(): - stream_hash = stream_info['sources']['lbry_sd_hash'] - else: - stream_hash = stream_info['stream_hash'] - - d = self._get_lbry_file_by_sd_hash(stream_hash) - def _add_results(l): - if l: - if os.path.isfile(os.path.join(self.download_directory, l.file_name)): - return defer.succeed((stream_info, l)) - return defer.succeed((stream_info, None)) - d.addCallback(_add_results) - return d - - def _wait_on_lbry_file(f): - if os.path.isfile(os.path.join(self.download_directory, f.file_name)): - written_file = file(os.path.join(self.download_directory, f.file_name)) - written_file.seek(0, os.SEEK_END) - written_bytes = written_file.tell() - written_file.close() - else: - written_bytes = False - - if not written_bytes: - d = defer.succeed(None) - d.addCallback(lambda _: reactor.callLater(1, _wait_on_lbry_file, f)) - return d - else: - return defer.succeed(_disp_file(f)) - - def _disp_file(f): - file_path = os.path.join(self.download_directory, f.file_name) - log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.sd_hash) + " --> " + file_path) - return f - - def _get_stream(stream_info): - def _wait_for_write(): - try: - if os.path.isfile(os.path.join(self.download_directory, self.streams[name].downloader.file_name)): - written_file = file(os.path.join(self.download_directory, self.streams[name].downloader.file_name)) - written_file.seek(0, os.SEEK_END) - written_bytes = written_file.tell() - written_file.close() - else: - written_bytes = False - except: - written_bytes = False - - if not written_bytes: - d = defer.succeed(None) - d.addCallback(lambda _: reactor.callLater(1, _wait_for_write)) - return d - else: - return defer.succeed(None) - - self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet, - self.lbry_file_manager, max_key_fee=self.max_key_fee, - data_rate=self.data_rate, timeout=timeout, - download_directory=download_directory, file_name=file_name) - d = self.streams[name].start(stream_info, name) - if wait_for_write: - d.addCallback(lambda _: _wait_for_write()) - d.addCallback(lambda _: self.streams[name].downloader) - - return d + helper = _DownloadNameHelper( + self, name, timeout, download_directory, file_name, wait_for_write) if not stream_info: self.waiting_on[name] = True d = self._resolve_name(name) else: d = defer.succeed(stream_info) - d.addCallback(_setup_stream) - d.addCallback(lambda (stream_info, lbry_file): _get_stream(stream_info) if not lbry_file else _wait_on_lbry_file(lbry_file)) + d.addCallback(helper._setup_stream) + d.addCallback(helper.wait_or_get_stream) if not stream_info: - d.addCallback(_remove_from_wait) + d.addCallback(helper._remove_from_wait) + return d + + def add_stream(self, name, timeout, download_directory, file_name, stream_info): + """Makes, adds and starts a stream""" + self.streams[name] = GetStream(self.sd_identifier, + self.session, + self.session.wallet, + self.lbry_file_manager, + max_key_fee=self.max_key_fee, + data_rate=self.data_rate, + timeout=timeout, + download_directory=download_directory, + file_name=file_name) + d = self.streams[name].start(stream_info, name) return d def _get_long_count_timestamp(self): @@ -2273,3 +2213,92 @@ def get_output_callback(params): 'path': os.path.join(params.download_directory, l.file_name) } return callback + + +class _DownloadNameHelper(object): + def __init__(self, daemon, name, timeout=DEFAULT_TIMEOUT, download_directory=None, + file_name=None, wait_for_write=True): + self.daemon = daemon + self.name = name + self.timeout = timeout + if not download_directory or not os.path.isdir(download_directory): + self.download_directory = daemon.download_directory + else: + self.download_directory = download_directory + self.file_name = file_name + self.wait_for_write = wait_for_write + + def _setup_stream(self, stream_info): + stream_hash = get_sd_hash(stream_info) + d = self.daemon._get_lbry_file_by_sd_hash(stream_hash) + d.addCallback(self._add_results_callback(stream_info)) + return d + + def _add_results_callback(self, stream_info): + def add_results(l): + if l: + if os.path.isfile(os.path.join(self.download_directory, l.file_name)): + return defer.succeed((stream_info, l)) + return defer.succeed((stream_info, None)) + return add_results + + def wait_or_get_stream(self, args): + stream_info, lbry_file = args + if lbry_file: + return self._get_stream(stream_info) + else: + return self._wait_on_lbry_file(lbry_file) + + def _get_stream(self, stream_info): + d = self.daemon.add_stream( + self.name, self.timeout, self.download_directory, self.file_name, stream_info) + if self.wait_for_write: + d.addCallback(lambda _: self._wait_for_write()) + d.addCallback(lambda _: self.daemon.streams[self.name].downloader) + return d + + def _wait_for_write(self): + file_name = self.daemon.streams[self.name].downloader.file_name + written_bytes = self.get_written_bytes(file_name) + d = defer.succeed(None) + if not written_bytes: + d.addCallback(lambda _: reactor.callLater(1, self._wait_for_write)) + return d + + def _wait_on_lbry_file(self, f): + written_bytes = self.get_written_bytes(f.file_name) + if not written_bytes: + d = defer.succeed(None) + d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f)) + return d + else: + return defer.succeed(self._disp_file(f)) + + def get_written_bytes(self, file_name): + try: + file_path = os.path.join(self.download_directory, file_name) + if os.path.isfile(file_path): + written_file = file(file_path) + written_file.seek(0, os.SEEK_END) + written_bytes = written_file.tell() + written_file.close() + else: + written_bytes = False + except Exception: + writen_bytes = False + return written_bytes + + def _disp_file(self, f): + file_path = os.path.join(self.download_directory, f.file_name) + log.info("[%s] Already downloaded: %s --> %s", datetime.now(), f.sd_hash, file_path) + return f + + def _remove_from_wait(self, r): + del self.daemon.waiting_on[self.name] + return r + + + + + + From 8743decd17cf5a8398ab152e911d8b9bb0bb82a3 Mon Sep 17 00:00:00 2001 From: Harwinder Date: Fri, 15 Jul 2016 20:01:25 +0200 Subject: [PATCH 10/36] Need to have git installed. You cant execute the code that follows after if you havent installed git already. --- FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FAQ.md b/FAQ.md index 6336272da..091acd711 100644 --- a/FAQ.md +++ b/FAQ.md @@ -12,7 +12,7 @@ You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio On Ubuntu or Mint you can install the prerequisites and lbrynet by running - sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip + sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git git clone https://github.com/lbryio/lbry.git cd lbry sudo python setup.py install From 172f275bc70a696f491d62d654304f4402be831f Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sat, 16 Jul 2016 09:24:27 -0500 Subject: [PATCH 11/36] Refactor _resolve_name. Continue using the delegation/helper pattern for the daemon. --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 86 ++++++++++++++++++---------- 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index fad2583bd..828cbd65f 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1042,38 +1042,16 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _resolve_name(self, name, force_refresh=False): - def _cache_stream_info(stream_info): - def _add_txid(txid): - self.name_cache[name]['txid'] = txid - return defer.succeed(None) + """Resolves a name. Checks the cache first before going out to the blockchain. - self.name_cache[name] = {'claim_metadata': stream_info, 'timestamp': self._get_long_count_timestamp()} - d = self.session.wallet.get_txid_for_name(name) - d.addCallback(_add_txid) - d.addCallback(lambda _: self._update_claim_cache()) - d.addCallback(lambda _: self.name_cache[name]['claim_metadata']) - - return d - - if not force_refresh: - if name in self.name_cache.keys(): - if (self._get_long_count_timestamp() - self.name_cache[name]['timestamp']) < self.cache_time: - log.info("[" + str(datetime.now()) + "] Returning cached stream info for lbry://" + name) - d = defer.succeed(self.name_cache[name]['claim_metadata']) - else: - log.info("[" + str(datetime.now()) + "] Refreshing stream info for lbry://" + name) - d = self.session.wallet.get_stream_info_for_name(name) - d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) - else: - log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) - d = self.session.wallet.get_stream_info_for_name(name) - d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) - else: - log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) - d = self.session.wallet.get_stream_info_for_name(name) - d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) - - return d + Args: + name: the lbry:// to resolve + force_refresh: if True, always go out to the blockchain to resolve. + """ + if name.startswith('lbry://'): + raise ValueError('name %s should not start with lbry://') + helper = _ResolveNameHelper(self, name, force_refresh) + return helper.get_deferred() def _delete_lbry_file(self, lbry_file, delete_file=True): d = self.lbry_file_manager.delete_lbry_file(lbry_file) @@ -2297,8 +2275,54 @@ class _DownloadNameHelper(object): del self.daemon.waiting_on[self.name] return r +class _ResolveNameHelper(object): + def __init__(self, daemon, name, force_refresh): + self.daemon = daemon + self.name = name + self.force_refresh = force_refresh + def get_deferred(self): + if self.need_fresh_stream(): + log.info("Resolving stream info for lbry://%s", self.name) + d = self.wallet.get_stream_info_for_name(self.name) + d.addCallbacks(self._cache_stream_info, lambda _: defer.fail(UnknownNameError)) + else: + log.info("Returning cached stream info for lbry://%s", self.name) + d = defer.succeed(self.name_data['claim_metadata']) + return d + @property + def name_data(self): + return self.daemon.name_cache[self.name] + @property + def wallet(self): + return self.daemon.session.wallet + def now(self): + return self.daemon._get_long_count_timestamp() + def _add_txid(self, txid): + self.name_data['txid'] = txid + return defer.succeed(None) + + def _cache_stream_info(self, stream_info): + self.daemon.name_cache[self.name] = { + 'claim_metadata': stream_info, + 'timestamp': self.now() + } + d = self.wallet.get_txid_for_name(self.name) + d.addCallback(self._add_txid) + d.addCallback(lambda _: self.daemon._update_claim_cache()) + d.addCallback(lambda _: self.name_data['claim_metadata']) + return d + + def need_fresh_stream(self): + return self.force_refresh or not self.is_in_cache() or self.is_cached_name_expired() + + def is_in_cache(self): + return self.name in self.daemon.name_cache + + def is_cached_name_expired(self): + time_in_cache = self.now() - self.name_data['timestamp'] + return time_in_cache >= self.daemon.cache_time From 8ec10e23048b4b292ba1d6d949150fe9cd3ba23c Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sun, 17 Jul 2016 13:43:35 -0500 Subject: [PATCH 12/36] emergency fix: force lbrycrd to be the default wallet Ignore save settings for wallet type and use the default wallet (lbrycrd) instead. lbryum can be used but only if specified on the command line. --- lbrynet/conf.py | 4 ++-- lbrynet/lbrynet_daemon/LBRYDaemon.py | 35 +++++++++++++++++++++------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 9ead15f4d..b3ab58819 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -38,7 +38,7 @@ API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRES UI_ADDRESS = "http://%s:%i" % (API_INTERFACE, API_PORT) PROTOCOL_PREFIX = "lbry" -DEFAULT_WALLET = "lbryum" +DEFAULT_WALLET = "lbrycrd" WALLET_TYPES = ["lbryum", "lbrycrd"] DEFAULT_TIMEOUT = 30 DEFAULT_MAX_SEARCH_RESULTS = 25 @@ -49,4 +49,4 @@ DEFAULT_UI_BRANCH = "master" SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type'] -OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] \ No newline at end of file +OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 8e2c98cc5..914f21073 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -159,6 +159,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbryum_version = None self.ui_version = None self.ip = None + # TODO: this is confusing to set here, and then to be reset below. self.wallet_type = wallet_type self.first_run = None self.log_file = lbrynet_log @@ -264,13 +265,30 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.search_timeout = self.session_settings['search_timeout'] self.download_timeout = self.session_settings['download_timeout'] self.max_search_results = self.session_settings['max_search_results'] - if self.session_settings['wallet_type'] in WALLET_TYPES and not wallet_type: - self.wallet_type = self.session_settings['wallet_type'] - log.info("Using wallet type %s from config" % self.wallet_type) - else: + #### + # + # Ignore the saved wallet type. Some users will have their wallet type + # saved as lbryum and we want wallets to be lbrycrd unless explicitly + # set on the command line to be lbryum. + # + # if self.session_settings['wallet_type'] in WALLET_TYPES and not wallet_type: + # self.wallet_type = self.session_settings['wallet_type'] + # log.info("Using wallet type %s from config" % self.wallet_type) + # else: + # self.wallet_type = wallet_type + # self.session_settings['wallet_type'] = wallet_type + # log.info("Using wallet type %s specified from command line" % self.wallet_type) + # + # Instead, if wallet is not set on the command line, default to the default wallet + # + if wallet_type: + log.info("Using wallet type %s specified from command line", wallet_type) self.wallet_type = wallet_type - self.session_settings['wallet_type'] = wallet_type - log.info("Using wallet type %s specified from command line" % self.wallet_type) + else: + log.info("Using the default wallet type %s", DEFAULT_WALLET) + self.wallet_type = DEFAULT_WALLET + # + #### self.delete_blobs_on_remove = self.session_settings['delete_blobs_on_remove'] self.peer_port = self.session_settings['peer_port'] self.dht_node_port = self.session_settings['dht_node_port'] @@ -877,7 +895,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d def get_wallet(): - if self.wallet_type == "lbrycrd": #force lbrycrd wallet no matter what while lbryum is down + if self.wallet_type == "lbrycrd": log.info("Using lbrycrd wallet") d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.wallet_dir, wallet_conf=self.lbrycrd_conf, lbrycrdd_path=self.lbrycrdd_path)) @@ -888,7 +906,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): log.info("Using PTC wallet") d = defer.succeed(PTCWallet(self.db_dir)) else: - log.info("Requested unknown wallet '%s', using default lbryum" % self.wallet_type) + # TODO: should fail here. Can't switch to lbrycrd because the wallet_dir, conf and path won't be set + log.info("Requested unknown wallet '%s', using default lbryum", self.wallet_type) d = defer.succeed(LBRYumWallet(self.db_dir)) d.addCallback(lambda wallet: {"wallet": wallet}) From 5ebcc01bcca76b62999bbc0133fdd62b98176123 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sun, 17 Jul 2016 14:10:11 -0500 Subject: [PATCH 13/36] Bump version: 0.3.6 -> 0.3.7 --- .bumpversion.cfg | 2 +- lbrynet/__init__.py | 2 +- packaging/ubuntu/lbry.desktop | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cb7f8ec7f..0848536a0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.6 +current_version = 0.3.7 commit = True tag = True message = Bump version: {current_version} -> {new_version} diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index f106a1889..a0576d5af 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -4,5 +4,5 @@ log = logging.getLogger(__name__) logging.getLogger(__name__).addHandler(logging.NullHandler()) log.setLevel(logging.ERROR) -__version__ = "0.3.6" +__version__ = "0.3.7" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index 53408274c..1b0f41790 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.3.6 +Version=0.3.7 Name=LBRY Comment=The world's first user-owned content marketplace Icon=lbry From 97efd63c99ba50976d2eac9cbbd63a737e7b13f0 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sun, 17 Jul 2016 19:19:00 -0500 Subject: [PATCH 14/36] deprecate lbry-console script --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 01346fb88..60f800df2 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,7 @@ from setuptools import setup, find_packages base_dir = os.path.abspath(os.path.dirname(__file__)) -console_scripts = ['lbrynet-console = lbrynet.lbrynet_console.LBRYConsole:launch_lbry_console', - 'lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUploader:launch_stdin_uploader', +console_scripts = ['lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUploader:launch_stdin_uploader', 'lbrynet-stdout-downloader = lbrynet.lbrynet_console.LBRYStdoutDownloader:launch_stdout_downloader', 'lbrynet-create-network = lbrynet.create_network:main', 'lbrynet-launch-node = lbrynet.dht.node:main', From 5817b80c947b506ca042f4a7a9805b0a7849fcab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Job=20Evers=E2=80=90Meltzer?= Date: Sun, 17 Jul 2016 20:17:42 -0500 Subject: [PATCH 15/36] script to migrate private keys to lbrycrd (#79) * script to migrate private keys to lbrycrd * handle compressed keys too * validate private key actually gets added * add lbrycrdd and lbrycrd-cli checks * exit if no wallet path * move each imported address into the default account --- scripts/migrate_lbryum_to_lbrycrd.py | 113 +++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 scripts/migrate_lbryum_to_lbrycrd.py diff --git a/scripts/migrate_lbryum_to_lbrycrd.py b/scripts/migrate_lbryum_to_lbrycrd.py new file mode 100644 index 000000000..19391ba75 --- /dev/null +++ b/scripts/migrate_lbryum_to_lbrycrd.py @@ -0,0 +1,113 @@ +import argparse +import hashlib +import json +import subprocess +import sys + +import base58 + +from lbryum import SimpleConfig, Network +from lbryum.wallet import WalletStorage, Wallet +from lbryum.commands import known_commands, Commands +from lbryum import lbrycrd + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--wallet', help='path to lbryum wallet') + args = parser.parse_args() + + ensureCliIsOnPathAndServerIsRunning() + + wallet = getWallet(args.wallet) + addresses = wallet.addresses(True) + for addr in addresses[:-1]: + printBalance(wallet, addr) + saveAddr(wallet, addr) + # on the last one, rescan. Don't rescan early for sake of efficiency + addr = addresses[-1] + printBalance(wallet, addr) + saveAddr(wallet, addr, "true") + + +def ensureCliIsOnPathAndServerIsRunning(): + try: + output = subprocess.check_output(['lbrycrd-cli', 'getinfo']) + except OSError: + print 'Failed to run: lbrycrd-cli needs to be on the PATH' + exit(1) + except subprocess.CalledProcessError: + print 'Failed to run: could not connect to the lbrycrd server.' + print 'Make sure it is running and able to be connected to.' + print 'One way to do this is to run:' + print ' lbrycrdd -server -printtoconsole' + exit(1) + + +def validateAddress(addr): + raw_output = subprocess.check_output( + ['lbrycrd-cli', 'validateaddress', addr]) + output = json.loads(raw_output) + if not output['isvalid']: + raise Exception('Address {} is not valid'.format(addr)) + if not output['ismine']: + raise Exception('Address {} is not yours'.format(addr)) + + +def printBalance(wallet, addr): + balance = getBalance(wallet, addr) + print 'Importing private key for %s with balance %s' % (addr, balance) + + +def getBalance(wallet, addr): + return sum(wallet.get_addr_balance(addr)) + + +def getWallet(path=None): + if not path: + config = SimpleConfig() + path = config.get_wallet_path() + storage = WalletStorage(path) + if not storage.file_exists: + print "Failed to run: No wallet to migrate" + exit(1) + return Wallet(storage) + + +def saveAddr(wallet, addr, rescan="false"): + keys = wallet.get_private_key(addr, None) + assert len(keys) == 1, 'Address {} has {} keys. Expected 1'.format(addr, len(keys)) + key = keys[0] + # copied from lbrycrd.regenerate_key + b = lbrycrd.ASecretToSecret(key) + pkey = b[0:32] + is_compressed = lbrycrd.is_compressed(key) + wif = pkeyToWif(pkey, is_compressed) + subprocess.check_call( + ['lbrycrd-cli', 'importprivkey', wif, "lbryum import", rescan]) + validateAddress(addr) + # during the import the account gets set to the label, but lbry + # needs the address to be in the default account + subprocess.check_call(['lbrycrd-cli', 'setaccount', addr, '""']) + + +def pkeyToWif(pkey, compressed): + # Follow https://en.bitcoin.it/wiki/Wallet_import_format + # to convert from a private key to the wallet import format + prefix = '\x1c' + wif = prefix + pkey + if compressed: + wif += '\x01' + intermediate_checksum = hashlib.sha256(wif).digest() + checksum = hashlib.sha256(intermediate_checksum).digest() + wif = wif + checksum[:4] + return base58.b58encode(wif) + + +def wifToPkey(wif): + pkey = base58.b58decode(wif) + return pkey[1:-4] + + +if __name__ == '__main__': + sys.exit(main()) From a384960a7b7e154e822416c828b693535512932c Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sun, 17 Jul 2016 19:19:00 -0500 Subject: [PATCH 16/36] deprecate lbry-console script --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 01346fb88..60f800df2 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,7 @@ from setuptools import setup, find_packages base_dir = os.path.abspath(os.path.dirname(__file__)) -console_scripts = ['lbrynet-console = lbrynet.lbrynet_console.LBRYConsole:launch_lbry_console', - 'lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUploader:launch_stdin_uploader', +console_scripts = ['lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUploader:launch_stdin_uploader', 'lbrynet-stdout-downloader = lbrynet.lbrynet_console.LBRYStdoutDownloader:launch_stdout_downloader', 'lbrynet-create-network = lbrynet.create_network:main', 'lbrynet-launch-node = lbrynet.dht.node:main', From a86b4c4e58c32ddba8d5eb90733786f6c1518f93 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Mon, 18 Jul 2016 08:45:14 -0500 Subject: [PATCH 17/36] bug fix: use sys.exit() not exit() --- scripts/migrate_lbryum_to_lbrycrd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/migrate_lbryum_to_lbrycrd.py b/scripts/migrate_lbryum_to_lbrycrd.py index 19391ba75..da49bd7dd 100644 --- a/scripts/migrate_lbryum_to_lbrycrd.py +++ b/scripts/migrate_lbryum_to_lbrycrd.py @@ -35,13 +35,13 @@ def ensureCliIsOnPathAndServerIsRunning(): output = subprocess.check_output(['lbrycrd-cli', 'getinfo']) except OSError: print 'Failed to run: lbrycrd-cli needs to be on the PATH' - exit(1) + sys.exit(1) except subprocess.CalledProcessError: print 'Failed to run: could not connect to the lbrycrd server.' print 'Make sure it is running and able to be connected to.' print 'One way to do this is to run:' print ' lbrycrdd -server -printtoconsole' - exit(1) + sys.exit(1) def validateAddress(addr): @@ -70,7 +70,7 @@ def getWallet(path=None): storage = WalletStorage(path) if not storage.file_exists: print "Failed to run: No wallet to migrate" - exit(1) + sys.exit(1) return Wallet(storage) From 1a5cd9b06241e3b05fa386b96c4432207027ac20 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Mon, 18 Jul 2016 09:14:50 -0500 Subject: [PATCH 18/36] Forget about trying to set a label. Setting a label also sets the account, but the LBRY app only looks at the default "" account. Marking the imported keys would have been nice, but it doesn't work. --- scripts/migrate_lbryum_to_lbrycrd.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/migrate_lbryum_to_lbrycrd.py b/scripts/migrate_lbryum_to_lbrycrd.py index da49bd7dd..fdafacd6e 100644 --- a/scripts/migrate_lbryum_to_lbrycrd.py +++ b/scripts/migrate_lbryum_to_lbrycrd.py @@ -84,11 +84,8 @@ def saveAddr(wallet, addr, rescan="false"): is_compressed = lbrycrd.is_compressed(key) wif = pkeyToWif(pkey, is_compressed) subprocess.check_call( - ['lbrycrd-cli', 'importprivkey', wif, "lbryum import", rescan]) + ['lbrycrd-cli', 'importprivkey', wif, "", rescan]) validateAddress(addr) - # during the import the account gets set to the label, but lbry - # needs the address to be in the default account - subprocess.check_call(['lbrycrd-cli', 'setaccount', addr, '""']) def pkeyToWif(pkey, compressed): From 9127a816e3183f93df4db600b52f071010e254c5 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Mon, 18 Jul 2016 10:12:49 -0500 Subject: [PATCH 19/36] Update protobuf library Updating to protobuf-v3.0.0b3 because it fixes a bug here https://github.com/google/protobuf/blob/e841bac4fcf47f809e089a70d5f84ac37b3883df/python/google/protobuf/internal/python_message.py#L59 where six.moves is not always available. In particular, the system python on OSX has an old version of six which is without six.moves. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4423b5db..780abee9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum leveldb==0.193 miniupnpc==1.9 pbkdf2==1.3 -protobuf==3.0.0b2 +protobuf==3.0.0b3 pycrypto==2.6.1 python-bitcoinrpc==0.1 qrcode==5.2.2 From 78b4b3ffa8e06753b6a2237cd90ae8997d99a3ed Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Mon, 18 Jul 2016 11:30:21 -0500 Subject: [PATCH 20/36] revert default wallet back to lbryum --- lbrynet/conf.py | 2 +- lbrynet/lbrynet_daemon/LBRYDaemon.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index b3ab58819..6bf8df48c 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -38,7 +38,7 @@ API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRES UI_ADDRESS = "http://%s:%i" % (API_INTERFACE, API_PORT) PROTOCOL_PREFIX = "lbry" -DEFAULT_WALLET = "lbrycrd" +DEFAULT_WALLET = "lbryum" WALLET_TYPES = ["lbryum", "lbrycrd"] DEFAULT_TIMEOUT = 30 DEFAULT_MAX_SEARCH_RESULTS = 25 diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 914f21073..aaf504080 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -268,8 +268,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): #### # # Ignore the saved wallet type. Some users will have their wallet type - # saved as lbryum and we want wallets to be lbrycrd unless explicitly - # set on the command line to be lbryum. + # saved as lbrycrd and we want wallets to be lbryum unless explicitly + # set on the command line to be lbrycrd. # # if self.session_settings['wallet_type'] in WALLET_TYPES and not wallet_type: # self.wallet_type = self.session_settings['wallet_type'] From f1b4c6a1ec2e4c63bae8bc89953b40eafb241bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Job=20Evers=E2=80=90Meltzer?= Date: Mon, 18 Jul 2016 14:50:05 -0500 Subject: [PATCH 21/36] Cache pyobjc (#87) * cache pyobjc modules as wheels --- .travis.yml | 1 + packaging/osx/lbry-osx-app/setup_app.sh | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cead1a849..ca2557860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ cache: directories: - $HOME/.cache/pip - $HOME/Library/Caches/pip + - $TRAVIS_BUILD_DIR/cache/wheel before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi diff --git a/packaging/osx/lbry-osx-app/setup_app.sh b/packaging/osx/lbry-osx-app/setup_app.sh index 5630a1e3f..ad3ca8449 100755 --- a/packaging/osx/lbry-osx-app/setup_app.sh +++ b/packaging/osx/lbry-osx-app/setup_app.sh @@ -5,9 +5,11 @@ set -o xtrace DEST=`pwd` tmp="${DEST}/build" +ON_TRAVIS=false rm -rf build dist LBRY.app +pip install wheel # the default py2app (v0.9) has a bug that is fixed in the head of /metachris/py2app pip install git+https://github.com/metachris/py2app pip install jsonrpc @@ -23,6 +25,7 @@ if [ -z ${TRAVIS_BUILD_DIR+x} ]; then LBRY="${tmp}/lbry" else # building on travis + ON_TRAVIS=true cd ${TRAVIS_BUILD_DIR} LBRY=${TRAVIS_BUILD_DIR} fi @@ -45,7 +48,25 @@ codesign -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/ codesign --deep -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/MacOS/LBRYURIHandler" codesign -vvvv "${DEST}/dist/LBRYURIHandler.app" -pip install certifi pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork +pip install certifi +MODULES="pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork" +if [ ${ON_TRAVIS} = true ]; then + WHEEL_DIR="${TRAVIS_BUILD_DIR}/cache/wheel" + mkdir -p "${WHEEL_DIR}" + # mapping from the package name to the + # actual built wheel file is surprisingly + # hard so instead of checking for the existance + # of each wheel, we mark with a file when they've all been + # built and skip when that file exists + if [ ! -f "${WHEEL_DIR}"/finished ]; then + pip wheel -w "${WHEEL_DIR}" ${MODULES} + touch "${WHEEL_DIR}"/finished + fi + pip install "${WHEEL_DIR}"/*.whl +else + pip install $MODULES +fi + # add lbrycrdd as a resource. Following # http://stackoverflow.com/questions/11370012/can-executables-made-with-py2app-include-other-terminal-scripts-and-run-them From 4adec39dcef895ded59b3ef0a1071d8af6e718a6 Mon Sep 17 00:00:00 2001 From: Job Evers Date: Mon, 18 Jul 2016 16:00:52 -0500 Subject: [PATCH 22/36] Bump version: 0.3.7 -> 0.3.8 --- .bumpversion.cfg | 2 +- lbrynet/__init__.py | 2 +- packaging/ubuntu/lbry.desktop | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0848536a0..b34ee8dd9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.7 +current_version = 0.3.8 commit = True tag = True message = Bump version: {current_version} -> {new_version} diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index a0576d5af..49bd262cf 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -4,5 +4,5 @@ log = logging.getLogger(__name__) logging.getLogger(__name__).addHandler(logging.NullHandler()) log.setLevel(logging.ERROR) -__version__ = "0.3.7" +__version__ = "0.3.8" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index 1b0f41790..1c28d3553 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.3.7 +Version=0.3.8 Name=LBRY Comment=The world's first user-owned content marketplace Icon=lbry From a54166a27cb1cbabfc516196dfeb2eebb857b6b9 Mon Sep 17 00:00:00 2001 From: Alex Liebowitz Date: Wed, 20 Jul 2016 02:36:55 -0400 Subject: [PATCH 23/36] Add support for file uploads (for Publish page) --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 10 + lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 6 +- lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 205 ++++++++++++++++++++ 3 files changed, 219 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index aaf504080..379209634 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -170,6 +170,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.streams = {} self.known_dht_nodes = KNOWN_DHT_NODES self.first_run_after_update = False + self.uploaded_temp_files = [] if os.name == "nt": from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle @@ -708,6 +709,13 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: return defer.succeed(None) + def _clean_up_temp_files(self): + for path in self.uploaded_temp_files: + try: + os.remove(path) + except OSError: + pass + def _shutdown(self): log.info("Closing lbrynet session") log.info("Status at time of shutdown: " + self.startup_status[0]) @@ -720,6 +728,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): if self.lbry_ui_manager.update_checker.running: self.lbry_ui_manager.update_checker.stop() + self._clean_up_temp_files() + d = self._upload_log(log_type="close", exclude_previous=False if self.first_run else True) d.addCallback(lambda _: self._stop_server()) d.addErrback(lambda err: True) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 309570873..dff9784ee 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -12,7 +12,7 @@ from twisted.web import server from twisted.internet import reactor, defer from jsonrpc.proxy import JSONRPCProxy -from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer +from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, \ DEFAULT_WALLET, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME @@ -109,7 +109,9 @@ def start(): if args.launchui: d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) - reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE) + lbrynet_server = server.Site(lbry.root) + lbrynet_server.requestFactory = LBRYDaemonRequest + reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE) reactor.run() if not args.logtoconsole and not args.quiet: diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index 74906da0b..b28520471 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -4,6 +4,10 @@ import shutil import json import sys import mimetypes +import mimetools +import tempfile +import time +import cgi from datetime import datetime from appdirs import user_data_dir @@ -29,6 +33,184 @@ handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, ba log.addHandler(handler) log.setLevel(logging.INFO) +class LBRYDaemonRequest(server.Request): + """ + For LBRY specific request functionality. Currently just provides + handling for large multipart POST requests, taken from here: + http://sammitch.ca/2013/07/handling-large-requests-in-twisted/ + + For multipart POST requests, this populates self.args with temp + file objects instead of strings. Note that these files don't auto-delete + on close because we want to be able to move and rename them. + + """ + + # max amount of memory to allow any ~single~ request argument [ie: POSTed file] + # note: this value seems to be taken with a grain of salt, memory usage may spike + # FAR above this value in some cases. + # eg: set the memory limit to 5 MB, write 2 blocks of 4MB, mem usage will + # have spiked to 8MB before the data is rolled to disk after the + # second write completes. + memorylimit = 1024*1024*100 + + # enable/disable debug logging + do_log = True + + # re-defined only for debug/logging purposes + def gotLength(self, length): + if self.do_log: + print '%f Headers received, Content-Length: %d' % (time.time(), length) + server.Request.gotLength(self, length) + + # re-definition of twisted.web.server.Request.requestreceived, the only difference + # is that self.parse_multipart() is used rather than cgi.parse_multipart() + def requestReceived(self, command, path, version): + from twisted.web.http import parse_qs + if self.do_log: + print '%f Request Received' % time.time() + print self.content + + self.content.seek(0,0) + self.args = {} + self.stack = [] + + self.method, self.uri = command, path + self.clientproto = version + x = self.uri.split(b'?', 1) + + if len(x) == 1: + self.path = self.uri + else: + self.path, argstring = x + self.args = parse_qs(argstring, 1) + + # cache the client and server information, we'll need this later to be + # serialized and sent with the request so CGIs will work remotely + self.client = self.channel.transport.getPeer() + self.host = self.channel.transport.getHost() + + # Argument processing + args = self.args + ctype = self.requestHeaders.getRawHeaders(b'content-type') + if ctype is not None: + ctype = ctype[0] + + if self.method == b"POST" and ctype: + mfd = b'multipart/form-data' + key, pdict = cgi.parse_header(ctype) + if key == b'application/x-www-form-urlencoded': + args.update(parse_qs(self.content.read(), 1)) + elif key == mfd: + try: + self.content.seek(0,0) + args.update(self.parse_multipart(self.content, pdict)) + #args.update(cgi.parse_multipart(self.content, pdict)) + + except KeyError as e: + if e.args[0] == b'content-disposition': + # Parse_multipart can't cope with missing + # content-dispostion headers in multipart/form-data + # parts, so we catch the exception and tell the client + # it was a bad request. + self.channel.transport.write( + b"HTTP/1.1 400 Bad Request\r\n\r\n") + self.channel.transport.loseConnection() + return + raise + + self.content.seek(0, 0) + + self.process() + + # re-definition of cgi.parse_multipart that uses a single temporary file to store + # data rather than storing 2 to 3 copies in various lists. + def parse_multipart(self, fp, pdict): + if self.do_log: + print '%f Parsing Multipart data: ' % time.time() + rewind = fp.tell() #save cursor + fp.seek(0,0) #reset cursor + + boundary = "" + if 'boundary' in pdict: + boundary = pdict['boundary'] + if not cgi.valid_boundary(boundary): + raise ValueError, ('Invalid boundary in multipart form: %r' + % (boundary,)) + + nextpart = "--" + boundary + lastpart = "--" + boundary + "--" + partdict = {} + terminator = "" + + while terminator != lastpart: + c_bytes = -1 + + data = tempfile.NamedTemporaryFile(delete=False) + if terminator: + # At start of next part. Read headers first. + headers = mimetools.Message(fp) + clength = headers.getheader('content-length') + if clength: + try: + c_bytes = int(clength) + except ValueError: + pass + if c_bytes > 0: + data.write(fp.read(c_bytes)) + # Read lines until end of part. + while 1: + line = fp.readline() + if not line: + terminator = lastpart # End outer loop + break + if line[:2] == "--": + terminator = line.strip() + if terminator in (nextpart, lastpart): + break + data.write(line) + # Done with part. + if data.tell() == 0: + continue + if c_bytes < 0: + # if a Content-Length header was not supplied with the MIME part + # then the trailing line break must be removed. + # we have data, read the last 2 bytes + rewind = min(2, data.tell()) + data.seek(-rewind, os.SEEK_END) + line = data.read(2) + if line[-2:] == "\r\n": + data.seek(-2, os.SEEK_END) + data.truncate() + elif line[-1:] == "\n": + data.seek(-1, os.SEEK_END) + data.truncate() + + line = headers['content-disposition'] + if not line: + continue + key, params = cgi.parse_header(line) + if key != 'form-data': + continue + if 'name' in params: + name = params['name'] + # kludge in the filename + if 'filename' in params: + fname_index = name + '_filename' + if fname_index in partdict: + partdict[fname_index].append(params['filename']) + else: + partdict[fname_index] = [params['filename']] + else: + # Unnamed parts are not returned at all. + continue + data.seek(0,0) + if name in partdict: + partdict[name].append(data) + else: + partdict[name] = [data] + + fp.seek(rewind) # Restore cursor + return partdict class LBRYindex(resource.Resource): def __init__(self, ui_dir): @@ -179,12 +361,35 @@ class HostedLBRYFile(resource.Resource): # call.addErrback(lambda err: log.info("Error: " + str(err))) # call.cancel() +class LBRYFileUpload(resource.Resource): + """ + Accepts a file sent via the file upload widget in the web UI, saves + it into a temporary dir, and responds with a JSON string containing + the path of the newly created file. + """ + + def __init__(self, api): + self._api = api + + def render_POST(self, request): + origfilename = request.args['file_filename'][0] + uploaded_file = request.args['file'][0] # Temp file created by request + + # Move to a new temporary dir and restore the original file name + newdirpath = tempfile.mkdtemp() + newpath = os.path.join(newdirpath, origfilename) + shutil.move(uploaded_file.name, newpath) + self._api.uploaded_temp_files.append(newpath) + + return json.dumps(newpath) + class LBRYDaemonServer(object): def _setup_server(self, wallet): self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active")) self._api = LBRYDaemon(self.root, wallet_type=wallet) self.root.putChild("view", HostedLBRYFile(self._api)) + self.root.putChild("upload", LBRYFileUpload(self._api)) self.root.putChild(API_ADDRESS, self._api) return defer.succeed(True) From 5c27b9eb90179400a5375a963c0a86ea93f8db14 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Sat, 16 Jul 2016 01:15:58 -0500 Subject: [PATCH 24/36] use asctime in logging format Instead of manually adding datetime.now() use asctime formatter to do it for us. --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 53 +++++++++++---------- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 16 ++++++- lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 10 ++-- lbrynet/lbrynet_daemon/LBRYDownloader.py | 2 +- lbrynet/lbrynet_daemon/LBRYPublisher.py | 4 +- 5 files changed, 50 insertions(+), 35 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index ceb8a87a7..888406e09 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -66,6 +66,11 @@ if not os.path.isdir(log_dir): lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) log = logging.getLogger(__name__) + +# TODO: configuring a logger on module import drastically reduces the +# amount of control the caller of this code has over logging +# +# Better would be to configure all logging at runtime. handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) log.addHandler(handler) log.setLevel(logging.INFO) @@ -466,7 +471,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _announce(): self.announced_startup = True self.startup_status = STARTUP_STAGES[5] - log.info("[" + str(datetime.now()) + "] Started lbrynet-daemon") + log.info("Started lbrynet-daemon") if len(self.startup_scripts): log.info("Scheduling scripts") reactor.callLater(3, self._run_scripts) @@ -488,7 +493,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: _announce()) return d - log.info("[" + str(datetime.now()) + "] Starting lbrynet-daemon") + log.info("Starting lbrynet-daemon") self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) @@ -553,14 +558,14 @@ class LBRYDaemon(jsonrpc.JSONRPC): s = socket.create_connection((host, 80), 2) self.connected_to_internet = True except: - log.info("[" + str(datetime.now()) + "] Internet connection not working") + log.info("Internet connection not working") self.connected_to_internet = False def _check_lbrynet_connection(self): def _log_success(): - log.info("[" + str(datetime.now()) + "] lbrynet connectivity test passed") + log.info("lbrynet connectivity test passed") def _log_failure(): - log.info("[" + str(datetime.now()) + "] lbrynet connectivity test failed") + log.info("lbrynet connectivity test failed") wonderfullife_sh = "6f3af0fa3924be98a54766aa2715d22c6c1509c3f7fa32566df4899a41f3530a9f97b2ecb817fa1dcbf1b30553aefaa7" d = download_sd_blob(self.session, wonderfullife_sh, self.session.base_payment_rate_manager) @@ -578,7 +583,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbryum_version = version return defer.succeed(None) except: - log.info("[" + str(datetime.now()) + "] Failed to get lbryum version from git") + log.info("Failed to get lbryum version from git") self.git_lbryum_version = None return defer.fail(None) @@ -593,7 +598,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbrynet_version = vr return defer.succeed(None) except: - log.info("[" + str(datetime.now()) + "] Failed to get lbrynet version from git") + log.info("Failed to get lbrynet version from git") self.git_lbrynet_version = None return defer.fail(None) @@ -1091,15 +1096,15 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d d.addCallback(lambda _: finish_deletion(lbry_file)) - d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Delete lbry file")) + d.addCallback(lambda _: log.info("Delete lbry file")) return d def _get_est_cost(self, name): def _check_est(d, name): if isinstance(d.result, float): - log.info("[" + str(datetime.now()) + "] Cost est for lbry://" + name + ": " + str(d.result) + "LBC") + log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") else: - log.info("[" + str(datetime.now()) + "] Timeout estimating cost for lbry://" + name + ", using key fee") + log.info("Timeout estimating cost for lbry://" + name + ", using key fee") d.cancel() return defer.succeed(None) @@ -1294,7 +1299,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Returns: true if daemon completed startup, otherwise false """ - log.info("[" + str(datetime.now()) + "] is_running: " + str(self.announced_startup)) + log.info("is_running: " + str(self.announced_startup)) if self.announced_startup: return self._render_response(True, OK_CODE) @@ -1332,7 +1337,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: r['message'] = "Catching up with the blockchain" r['progress'] = 0 - log.info("[" + str(datetime.now()) + "] daemon status: " + str(r)) + log.info("daemon status: " + str(r)) return self._render_response(r, OK_CODE) def jsonrpc_is_first_run(self): @@ -1345,7 +1350,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): True if first run, otherwise False """ - log.info("[" + str(datetime.now()) + "] Check if is first run") + log.info("Check if is first run") try: d = self.session.wallet.is_first_run() except: @@ -1365,7 +1370,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Startup message, such as first run notification """ - log.info("[" + str(datetime.now()) + "] Get startup notice") + log.info("Get startup notice") if self.first_run and not self.session.wallet.wallet_balance: return self._render_response(self.startup_message, OK_CODE) @@ -1405,7 +1410,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'lbryum_update_available': lbryum_version < self.git_lbryum_version } - log.info("[" + str(datetime.now()) + "] Get version info: " + json.dumps(msg)) + log.info("Get version info: " + json.dumps(msg)) return self._render_response(msg, OK_CODE) def jsonrpc_get_settings(self): @@ -1433,7 +1438,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'start_lbrycrdd': bool, """ - log.info("[" + str(datetime.now()) + "] Get daemon settings") + log.info("Get daemon settings") return self._render_response(self.session_settings, OK_CODE) def jsonrpc_set_settings(self, p): @@ -1454,7 +1459,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _log_settings_change(): - log.info("[" + str(datetime.now()) + "] Set daemon settings to " + json.dumps(self.session_settings)) + log.info("Set daemon settings to " + json.dumps(self.session_settings)) d = self._update_settings(p) d.addErrback(lambda err: log.info(err.getTraceback())) @@ -1497,7 +1502,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): balance, float """ - log.info("[" + str(datetime.now()) + "] Get balance") + log.info("Get balance") return self._render_response(float(self.session.wallet.wallet_balance), OK_CODE) def jsonrpc_stop(self): @@ -1727,7 +1732,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.DeferredList(ds) def _disp(results): - log.info('[' + str(datetime.now()) + '] Found ' + str(len(results)) + ' search results') + log.info('Found ' + str(len(results)) + ' search results') consolidated_results = [] for r in results: t = {} @@ -1741,7 +1746,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return consolidated_results - log.info('[' + str(datetime.now()) + '] Search nametrie: ' + search) + log.info('Search nametrie: ' + search) d = self.session.wallet.get_nametrie() d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) @@ -1829,7 +1834,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return server.failure def _disp(x): - log.info("[" + str(datetime.now()) + "] Abandoned name claim tx " + str(x)) + log.info("Abandoned name claim tx " + str(x)) return self._render_response(x, OK_CODE) d = defer.Deferred() @@ -1940,7 +1945,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _disp(address): - log.info("[" + str(datetime.now()) + "] Got new wallet address: " + address) + log.info("Got new wallet address: " + address) return defer.succeed(address) d = self.session.wallet.get_new_address() @@ -2132,7 +2137,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): exclude_previous = True if 'message' in p.keys(): - log.info("[" + str(datetime.now()) + "] Upload log message: " + str(p['message'])) + log.info("Upload log message: " + str(p['message'])) if 'force' in p.keys(): force = p['force'] @@ -2287,7 +2292,7 @@ class _DownloadNameHelper(object): def _disp_file(self, f): file_path = os.path.join(self.download_directory, f.file_name) - log.info("[%s] Already downloaded: %s --> %s", datetime.now(), f.sd_hash, file_path) + log.info("Already downloaded: %s --> %s", f.sd_hash, file_path) return f def _remove_from_wait(self, r): diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 309570873..cdde77c6b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -25,8 +25,13 @@ if not os.path.isdir(log_dir): os.mkdir(log_dir) lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) + +DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s" +DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT) + log = logging.getLogger(__name__) handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) +handler.setFormatter(DEFAULT_FORMATTER) log.addHandler(handler) log.setLevel(logging.INFO) @@ -57,6 +62,13 @@ def stop(): d.callback(None) +def configureConsoleLogger(): + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(DEFAULT_FORMATTER) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(level=logging.INFO) + + def start(): parser = argparse.ArgumentParser(description="Launch lbrynet-daemon") parser.add_argument("--wallet", @@ -75,7 +87,7 @@ def start(): args = parser.parse_args() if args.logtoconsole: - logging.basicConfig(level=logging.INFO) + configureConsoleLogger() args = parser.parse_args() @@ -121,4 +133,4 @@ def start(): return if __name__ == "__main__": - start() \ No newline at end of file + start() diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index 74906da0b..507eb025d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -79,7 +79,7 @@ class LBRYFileStreamer(object): def pauseProducing(self): self._paused = True - log.info("[" + str(datetime.now()) + "] Pausing producer") + log.info("Pausing producer") return defer.succeed(None) def resumeProducing(self): @@ -104,7 +104,7 @@ class LBRYFileStreamer(object): self._request.write(data) self._cursor += 1 - log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % + log.info("Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % (start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth)) self._sent_bytes = True @@ -117,12 +117,12 @@ class LBRYFileStreamer(object): self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data)) return defer.succeed(None) - log.info("[" + str(datetime.now()) + "] Resuming producer") + log.info("Resuming producer") self._paused = False self._deferred.addCallback(lambda _: _check_for_new_data()) def stopProducing(self): - log.info("[" + str(datetime.now()) + "] Stopping producer") + log.info("Stopping producer") self._stopped = True # self._fileObject.close() self._deferred.addErrback(lambda err: err.trap(defer.CancelledError)) @@ -147,7 +147,7 @@ class HostedLBRYFile(resource.Resource): # # range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-') # start, stop = int(range_header[0]), range_header[1] - # log.info("[" + str(datetime.now()) + "] GET range %s-%s" % (start, stop)) + # log.info("GET range %s-%s" % (start, stop)) # path = os.path.join(self._api.download_directory, stream.file_name) # # d = stream.get_total_bytes() diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index c72444c70..9187cc56d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -147,6 +147,6 @@ class GetStream(object): d = _pay_key_fee() self.downloader = downloader self.download_path = os.path.join(downloader.download_directory, downloader.file_name) - d.addCallback(lambda _: log.info("[%s] Downloading %s --> %s" % (datetime.now(), self.stream_hash, self.downloader.file_name))) + d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name)) d.addCallback(lambda _: self.downloader.start()) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index c0658bfec..bd5c50e4b 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -48,9 +48,7 @@ class Publisher(object): def start(self, name, file_path, bid, metadata, fee=None, sources={}): def _show_result(): - - message = "[%s] Published %s --> lbry://%s txid: %s" % (datetime.now(), self.file_name, self.publish_name, self.txid) - log.info(message) + log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) return defer.succeed(self.txid) self.publish_name = name From 55f47a2b1b552f6272a7cd62815ea038bd892517 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 17:32:15 -0400 Subject: [PATCH 25/36] lbryum catchup --- lbrynet/core/LBRYWallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 0bf99f7a5..5088dc9d2 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -996,7 +996,7 @@ class LBRYumWallet(LBRYWallet): blockchain_caught_d = defer.Deferred() def check_caught_up(): - local_height = self.network.get_local_height() + local_height = self.network.get_catchup_progress() remote_height = self.network.get_server_height() if remote_height != 0 and remote_height - local_height <= 5: From c232743963d87082581116bbb721095bf9b9d292 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:13:44 -0400 Subject: [PATCH 26/36] fix LBRYumWallet._do_send_many --- lbrynet/core/LBRYWallet.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 5088dc9d2..4a8a4772f 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -1122,11 +1122,10 @@ class LBRYumWallet(LBRYWallet): def _do_send_many(self, payments_to_send): log.warning("Doing send many. payments to send: %s", str(payments_to_send)) - outputs = [(TYPE_ADDRESS, address, int(amount*COIN)) for address, amount in payments_to_send.iteritems()] - d = threads.deferToThread(self.wallet.mktx, outputs, None, self.config) - d.addCallback(lambda tx: threads.deferToThread(self.wallet.sendtx, tx)) - d.addCallback(self._save_wallet) - return d + cmd = known_commands['paytomanyandsend'] + func = getattr(self.cmd_runner, cmd.name) + # outputs = [(address, amount) for address, amount in payments_to_send.iteritems()] + return threads.deferToThread(func, payments_to_send.iteritems()) def _get_value_for_name(self, name): cmd = known_commands['getvalueforname'] From 0bee2689d1f276f5f9c86d66ba567fe85f46b766 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:17:01 -0400 Subject: [PATCH 27/36] download fixes --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index ceb8a87a7..295d91622 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1599,13 +1599,13 @@ class LBRYDaemon(jsonrpc.JSONRPC): # can spec what parameters it expects and how to set default values timeout = p.get('timeout', self.download_timeout) download_directory = p.get('download_directory', self.download_directory) - file_name = p.get('file_name') - stream_info = p.get('stream_info') + file_name = p.get('file_name', None) + stream_info = p.get('stream_info', None) sd_hash = get_sd_hash(stream_info) wait_for_write = p.get('wait_for_write', True) name = p.get('name') return Parameters( - timout=timeout, + timeout=timeout, download_directory=download_directory, file_name=file_name, stream_info=stream_info, @@ -1633,13 +1633,14 @@ class LBRYDaemon(jsonrpc.JSONRPC): return server.failure if params.name in self.waiting_on: return server.failure + d = self._download_name(name=params.name, timeout=params.timeout, download_directory=params.download_directory, stream_info=params.stream_info, file_name=params.file_name, wait_for_write=params.wait_for_write) - d.addCallback(get_output_callback(params)) + # d.addCallback(get_output_callback(params)) d.addCallback(lambda message: self._render_response(message, OK_CODE)) return d @@ -2263,13 +2264,15 @@ class _DownloadNameHelper(object): return d def _wait_on_lbry_file(self, f): - written_bytes = self.get_written_bytes(f.file_name) - if not written_bytes: - d = defer.succeed(None) - d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f)) - return d - else: - return defer.succeed(self._disp_file(f)) + if f is not None: + written_bytes = self.get_written_bytes(f.file_name) + if written_bytes: + return defer.succeed(self._disp_file(f)) + + d = defer.succeed(None) + d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f)) + return d + def get_written_bytes(self, file_name): try: From 7e7981855580476f04940fcf59797dd837961443 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:24:49 -0400 Subject: [PATCH 28/36] Revert "Merge branch 'master' into lbryum-catchup" This reverts commit a51576b250e58457ed001b9be94d959249fc9d3d, reversing changes made to 0bee2689d1f276f5f9c86d66ba567fe85f46b766. --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 53 ++++++++++----------- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 16 +------ lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 10 ++-- lbrynet/lbrynet_daemon/LBRYDownloader.py | 2 +- lbrynet/lbrynet_daemon/LBRYPublisher.py | 4 +- 5 files changed, 35 insertions(+), 50 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 3cd656644..295d91622 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -66,11 +66,6 @@ if not os.path.isdir(log_dir): lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) log = logging.getLogger(__name__) - -# TODO: configuring a logger on module import drastically reduces the -# amount of control the caller of this code has over logging -# -# Better would be to configure all logging at runtime. handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) log.addHandler(handler) log.setLevel(logging.INFO) @@ -471,7 +466,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _announce(): self.announced_startup = True self.startup_status = STARTUP_STAGES[5] - log.info("Started lbrynet-daemon") + log.info("[" + str(datetime.now()) + "] Started lbrynet-daemon") if len(self.startup_scripts): log.info("Scheduling scripts") reactor.callLater(3, self._run_scripts) @@ -493,7 +488,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: _announce()) return d - log.info("Starting lbrynet-daemon") + log.info("[" + str(datetime.now()) + "] Starting lbrynet-daemon") self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) @@ -558,14 +553,14 @@ class LBRYDaemon(jsonrpc.JSONRPC): s = socket.create_connection((host, 80), 2) self.connected_to_internet = True except: - log.info("Internet connection not working") + log.info("[" + str(datetime.now()) + "] Internet connection not working") self.connected_to_internet = False def _check_lbrynet_connection(self): def _log_success(): - log.info("lbrynet connectivity test passed") + log.info("[" + str(datetime.now()) + "] lbrynet connectivity test passed") def _log_failure(): - log.info("lbrynet connectivity test failed") + log.info("[" + str(datetime.now()) + "] lbrynet connectivity test failed") wonderfullife_sh = "6f3af0fa3924be98a54766aa2715d22c6c1509c3f7fa32566df4899a41f3530a9f97b2ecb817fa1dcbf1b30553aefaa7" d = download_sd_blob(self.session, wonderfullife_sh, self.session.base_payment_rate_manager) @@ -583,7 +578,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbryum_version = version return defer.succeed(None) except: - log.info("Failed to get lbryum version from git") + log.info("[" + str(datetime.now()) + "] Failed to get lbryum version from git") self.git_lbryum_version = None return defer.fail(None) @@ -598,7 +593,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbrynet_version = vr return defer.succeed(None) except: - log.info("Failed to get lbrynet version from git") + log.info("[" + str(datetime.now()) + "] Failed to get lbrynet version from git") self.git_lbrynet_version = None return defer.fail(None) @@ -1096,15 +1091,15 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d d.addCallback(lambda _: finish_deletion(lbry_file)) - d.addCallback(lambda _: log.info("Delete lbry file")) + d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Delete lbry file")) return d def _get_est_cost(self, name): def _check_est(d, name): if isinstance(d.result, float): - log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") + log.info("[" + str(datetime.now()) + "] Cost est for lbry://" + name + ": " + str(d.result) + "LBC") else: - log.info("Timeout estimating cost for lbry://" + name + ", using key fee") + log.info("[" + str(datetime.now()) + "] Timeout estimating cost for lbry://" + name + ", using key fee") d.cancel() return defer.succeed(None) @@ -1299,7 +1294,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Returns: true if daemon completed startup, otherwise false """ - log.info("is_running: " + str(self.announced_startup)) + log.info("[" + str(datetime.now()) + "] is_running: " + str(self.announced_startup)) if self.announced_startup: return self._render_response(True, OK_CODE) @@ -1337,7 +1332,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: r['message'] = "Catching up with the blockchain" r['progress'] = 0 - log.info("daemon status: " + str(r)) + log.info("[" + str(datetime.now()) + "] daemon status: " + str(r)) return self._render_response(r, OK_CODE) def jsonrpc_is_first_run(self): @@ -1350,7 +1345,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): True if first run, otherwise False """ - log.info("Check if is first run") + log.info("[" + str(datetime.now()) + "] Check if is first run") try: d = self.session.wallet.is_first_run() except: @@ -1370,7 +1365,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Startup message, such as first run notification """ - log.info("Get startup notice") + log.info("[" + str(datetime.now()) + "] Get startup notice") if self.first_run and not self.session.wallet.wallet_balance: return self._render_response(self.startup_message, OK_CODE) @@ -1410,7 +1405,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'lbryum_update_available': lbryum_version < self.git_lbryum_version } - log.info("Get version info: " + json.dumps(msg)) + log.info("[" + str(datetime.now()) + "] Get version info: " + json.dumps(msg)) return self._render_response(msg, OK_CODE) def jsonrpc_get_settings(self): @@ -1438,7 +1433,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'start_lbrycrdd': bool, """ - log.info("Get daemon settings") + log.info("[" + str(datetime.now()) + "] Get daemon settings") return self._render_response(self.session_settings, OK_CODE) def jsonrpc_set_settings(self, p): @@ -1459,7 +1454,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _log_settings_change(): - log.info("Set daemon settings to " + json.dumps(self.session_settings)) + log.info("[" + str(datetime.now()) + "] Set daemon settings to " + json.dumps(self.session_settings)) d = self._update_settings(p) d.addErrback(lambda err: log.info(err.getTraceback())) @@ -1502,7 +1497,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): balance, float """ - log.info("Get balance") + log.info("[" + str(datetime.now()) + "] Get balance") return self._render_response(float(self.session.wallet.wallet_balance), OK_CODE) def jsonrpc_stop(self): @@ -1733,7 +1728,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.DeferredList(ds) def _disp(results): - log.info('Found ' + str(len(results)) + ' search results') + log.info('[' + str(datetime.now()) + '] Found ' + str(len(results)) + ' search results') consolidated_results = [] for r in results: t = {} @@ -1747,7 +1742,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return consolidated_results - log.info('Search nametrie: ' + search) + log.info('[' + str(datetime.now()) + '] Search nametrie: ' + search) d = self.session.wallet.get_nametrie() d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) @@ -1835,7 +1830,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return server.failure def _disp(x): - log.info("Abandoned name claim tx " + str(x)) + log.info("[" + str(datetime.now()) + "] Abandoned name claim tx " + str(x)) return self._render_response(x, OK_CODE) d = defer.Deferred() @@ -1946,7 +1941,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _disp(address): - log.info("Got new wallet address: " + address) + log.info("[" + str(datetime.now()) + "] Got new wallet address: " + address) return defer.succeed(address) d = self.session.wallet.get_new_address() @@ -2138,7 +2133,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): exclude_previous = True if 'message' in p.keys(): - log.info("Upload log message: " + str(p['message'])) + log.info("[" + str(datetime.now()) + "] Upload log message: " + str(p['message'])) if 'force' in p.keys(): force = p['force'] @@ -2295,7 +2290,7 @@ class _DownloadNameHelper(object): def _disp_file(self, f): file_path = os.path.join(self.download_directory, f.file_name) - log.info("Already downloaded: %s --> %s", f.sd_hash, file_path) + log.info("[%s] Already downloaded: %s --> %s", datetime.now(), f.sd_hash, file_path) return f def _remove_from_wait(self, r): diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index cdde77c6b..309570873 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -25,13 +25,8 @@ if not os.path.isdir(log_dir): os.mkdir(log_dir) lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) - -DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s" -DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT) - log = logging.getLogger(__name__) handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) -handler.setFormatter(DEFAULT_FORMATTER) log.addHandler(handler) log.setLevel(logging.INFO) @@ -62,13 +57,6 @@ def stop(): d.callback(None) -def configureConsoleLogger(): - handler = logging.StreamHandler(sys.stdout) - handler.setFormatter(DEFAULT_FORMATTER) - logging.getLogger().addHandler(handler) - logging.getLogger().setLevel(level=logging.INFO) - - def start(): parser = argparse.ArgumentParser(description="Launch lbrynet-daemon") parser.add_argument("--wallet", @@ -87,7 +75,7 @@ def start(): args = parser.parse_args() if args.logtoconsole: - configureConsoleLogger() + logging.basicConfig(level=logging.INFO) args = parser.parse_args() @@ -133,4 +121,4 @@ def start(): return if __name__ == "__main__": - start() + start() \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index 507eb025d..74906da0b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -79,7 +79,7 @@ class LBRYFileStreamer(object): def pauseProducing(self): self._paused = True - log.info("Pausing producer") + log.info("[" + str(datetime.now()) + "] Pausing producer") return defer.succeed(None) def resumeProducing(self): @@ -104,7 +104,7 @@ class LBRYFileStreamer(object): self._request.write(data) self._cursor += 1 - log.info("Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % + log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % (start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth)) self._sent_bytes = True @@ -117,12 +117,12 @@ class LBRYFileStreamer(object): self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data)) return defer.succeed(None) - log.info("Resuming producer") + log.info("[" + str(datetime.now()) + "] Resuming producer") self._paused = False self._deferred.addCallback(lambda _: _check_for_new_data()) def stopProducing(self): - log.info("Stopping producer") + log.info("[" + str(datetime.now()) + "] Stopping producer") self._stopped = True # self._fileObject.close() self._deferred.addErrback(lambda err: err.trap(defer.CancelledError)) @@ -147,7 +147,7 @@ class HostedLBRYFile(resource.Resource): # # range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-') # start, stop = int(range_header[0]), range_header[1] - # log.info("GET range %s-%s" % (start, stop)) + # log.info("[" + str(datetime.now()) + "] GET range %s-%s" % (start, stop)) # path = os.path.join(self._api.download_directory, stream.file_name) # # d = stream.get_total_bytes() diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 9187cc56d..c72444c70 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -147,6 +147,6 @@ class GetStream(object): d = _pay_key_fee() self.downloader = downloader self.download_path = os.path.join(downloader.download_directory, downloader.file_name) - d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name)) + d.addCallback(lambda _: log.info("[%s] Downloading %s --> %s" % (datetime.now(), self.stream_hash, self.downloader.file_name))) d.addCallback(lambda _: self.downloader.start()) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index bd5c50e4b..c0658bfec 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -48,7 +48,9 @@ class Publisher(object): def start(self, name, file_path, bid, metadata, fee=None, sources={}): def _show_result(): - log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) + + message = "[%s] Published %s --> lbry://%s txid: %s" % (datetime.now(), self.file_name, self.publish_name, self.txid) + log.info(message) return defer.succeed(self.txid) self.publish_name = name From 5eb7c64ec363656d7b373e7587bca1794555c9ca Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:30:57 -0400 Subject: [PATCH 29/36] revert get() refactor --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 386 +++++++++++---------------- 1 file changed, 159 insertions(+), 227 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 295d91622..aaf504080 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -138,11 +138,6 @@ OK_CODE = 200 REMOTE_SERVER = "www.google.com" -class Parameters(object): - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - class LBRYDaemon(jsonrpc.JSONRPC): """ LBRYnet daemon, a jsonrpc interface to lbry functions @@ -1018,37 +1013,97 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None, - file_name=None, stream_info=None, wait_for_write=True): + file_name=None, stream_info=None, wait_for_write=True): """ Add a lbry file to the file manager, start the download, and return the new lbry file. If it already exists in the file manager, return the existing lbry file """ - helper = _DownloadNameHelper( - self, name, timeout, download_directory, file_name, wait_for_write) + + if not download_directory: + download_directory = self.download_directory + elif not os.path.isdir(download_directory): + download_directory = self.download_directory + + def _remove_from_wait(r): + del self.waiting_on[name] + return r + + def _setup_stream(stream_info): + if 'sources' in stream_info.keys(): + stream_hash = stream_info['sources']['lbry_sd_hash'] + else: + stream_hash = stream_info['stream_hash'] + + d = self._get_lbry_file_by_sd_hash(stream_hash) + def _add_results(l): + if l: + if os.path.isfile(os.path.join(self.download_directory, l.file_name)): + return defer.succeed((stream_info, l)) + return defer.succeed((stream_info, None)) + d.addCallback(_add_results) + return d + + def _wait_on_lbry_file(f): + if os.path.isfile(os.path.join(self.download_directory, f.file_name)): + written_file = file(os.path.join(self.download_directory, f.file_name)) + written_file.seek(0, os.SEEK_END) + written_bytes = written_file.tell() + written_file.close() + else: + written_bytes = False + + if not written_bytes: + d = defer.succeed(None) + d.addCallback(lambda _: reactor.callLater(1, _wait_on_lbry_file, f)) + return d + else: + return defer.succeed(_disp_file(f)) + + def _disp_file(f): + file_path = os.path.join(self.download_directory, f.file_name) + log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.sd_hash) + " --> " + file_path) + return f + + def _get_stream(stream_info): + def _wait_for_write(): + try: + if os.path.isfile(os.path.join(self.download_directory, self.streams[name].downloader.file_name)): + written_file = file(os.path.join(self.download_directory, self.streams[name].downloader.file_name)) + written_file.seek(0, os.SEEK_END) + written_bytes = written_file.tell() + written_file.close() + else: + written_bytes = False + except: + written_bytes = False + + if not written_bytes: + d = defer.succeed(None) + d.addCallback(lambda _: reactor.callLater(1, _wait_for_write)) + return d + else: + return defer.succeed(None) + + self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet, + self.lbry_file_manager, max_key_fee=self.max_key_fee, + data_rate=self.data_rate, timeout=timeout, + download_directory=download_directory, file_name=file_name) + d = self.streams[name].start(stream_info, name) + if wait_for_write: + d.addCallback(lambda _: _wait_for_write()) + d.addCallback(lambda _: self.streams[name].downloader) + + return d if not stream_info: self.waiting_on[name] = True d = self._resolve_name(name) else: d = defer.succeed(stream_info) - d.addCallback(helper._setup_stream) - d.addCallback(helper.wait_or_get_stream) + d.addCallback(_setup_stream) + d.addCallback(lambda (stream_info, lbry_file): _get_stream(stream_info) if not lbry_file else _wait_on_lbry_file(lbry_file)) if not stream_info: - d.addCallback(helper._remove_from_wait) - return d - - def add_stream(self, name, timeout, download_directory, file_name, stream_info): - """Makes, adds and starts a stream""" - self.streams[name] = GetStream(self.sd_identifier, - self.session, - self.session.wallet, - self.lbry_file_manager, - max_key_fee=self.max_key_fee, - data_rate=self.data_rate, - timeout=timeout, - download_directory=download_directory, - file_name=file_name) - d = self.streams[name].start(stream_info, name) + d.addCallback(_remove_from_wait) return d def _get_long_count_timestamp(self): @@ -1061,16 +1116,38 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _resolve_name(self, name, force_refresh=False): - """Resolves a name. Checks the cache first before going out to the blockchain. + def _cache_stream_info(stream_info): + def _add_txid(txid): + self.name_cache[name]['txid'] = txid + return defer.succeed(None) - Args: - name: the lbry:// to resolve - force_refresh: if True, always go out to the blockchain to resolve. - """ - if name.startswith('lbry://'): - raise ValueError('name %s should not start with lbry://') - helper = _ResolveNameHelper(self, name, force_refresh) - return helper.get_deferred() + self.name_cache[name] = {'claim_metadata': stream_info, 'timestamp': self._get_long_count_timestamp()} + d = self.session.wallet.get_txid_for_name(name) + d.addCallback(_add_txid) + d.addCallback(lambda _: self._update_claim_cache()) + d.addCallback(lambda _: self.name_cache[name]['claim_metadata']) + + return d + + if not force_refresh: + if name in self.name_cache.keys(): + if (self._get_long_count_timestamp() - self.name_cache[name]['timestamp']) < self.cache_time: + log.info("[" + str(datetime.now()) + "] Returning cached stream info for lbry://" + name) + d = defer.succeed(self.name_cache[name]['claim_metadata']) + else: + log.info("[" + str(datetime.now()) + "] Refreshing stream info for lbry://" + name) + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) + else: + log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) + else: + log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) + + return d def _delete_lbry_file(self, lbry_file, delete_file=True): d = self.lbry_file_manager.delete_lbry_file(lbry_file) @@ -1593,55 +1670,65 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d - def _process_get_parameters(self, p): - """Extract info from input parameters and fill in default values for `get` call.""" - # TODO: this process can be abstracted s.t. each method - # can spec what parameters it expects and how to set default values - timeout = p.get('timeout', self.download_timeout) - download_directory = p.get('download_directory', self.download_directory) - file_name = p.get('file_name', None) - stream_info = p.get('stream_info', None) - sd_hash = get_sd_hash(stream_info) - wait_for_write = p.get('wait_for_write', True) - name = p.get('name') - return Parameters( - timeout=timeout, - download_directory=download_directory, - file_name=file_name, - stream_info=stream_info, - sd_hash=sd_hash, - wait_for_write=wait_for_write, - name=name - ) - def jsonrpc_get(self, p): - """Download stream from a LBRY uri. + """ + Download stream from a LBRY uri Args: 'name': name to download, string 'download_directory': optional, path to directory where file will be saved, string 'file_name': optional, a user specified name for the downloaded file 'stream_info': optional, specified stream info overrides name - 'timout': optional - 'wait_for_write': optional, defaults to True Returns: 'stream_hash': hex string 'path': path of download """ - params = self._process_get_parameters(p) - if not params.name: - return server.failure - if params.name in self.waiting_on: - return server.failure - d = self._download_name(name=params.name, - timeout=params.timeout, - download_directory=params.download_directory, - stream_info=params.stream_info, - file_name=params.file_name, - wait_for_write=params.wait_for_write) - # d.addCallback(get_output_callback(params)) - d.addCallback(lambda message: self._render_response(message, OK_CODE)) + if 'timeout' not in p.keys(): + timeout = self.download_timeout + else: + timeout = p['timeout'] + + if 'download_directory' not in p.keys(): + download_directory = self.download_directory + else: + download_directory = p['download_directory'] + + if 'file_name' in p.keys(): + file_name = p['file_name'] + else: + file_name = None + + if 'stream_info' in p.keys(): + stream_info = p['stream_info'] + if 'sources' in stream_info.keys(): + sd_hash = stream_info['sources']['lbry_sd_hash'] + else: + sd_hash = stream_info['stream_hash'] + else: + stream_info = None + + if 'wait_for_write' in p.keys(): + wait_for_write = p['wait_for_write'] + else: + wait_for_write = True + + if 'name' in p.keys(): + name = p['name'] + if p['name'] not in self.waiting_on.keys(): + d = self._download_name(name=name, timeout=timeout, download_directory=download_directory, + stream_info=stream_info, file_name=file_name, wait_for_write=wait_for_write) + d.addCallback(lambda l: {'stream_hash': sd_hash, + 'path': os.path.join(self.download_directory, l.file_name)} + if stream_info else + {'stream_hash': l.sd_hash, + 'path': os.path.join(self.download_directory, l.file_name)}) + d.addCallback(lambda message: self._render_response(message, OK_CODE)) + else: + d = server.failure + else: + d = server.failure + return d def jsonrpc_stop_lbry_file(self, p): @@ -2188,163 +2275,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = threads.deferToThread(subprocess.Popen, ['open', '-R', path]) else: # No easy way to reveal specific files on Linux, so just open the containing directory - d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.path.dirname(path)]) + d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.dirname(path)]) d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - - -def get_sd_hash(stream_info): - if not stream_info: - return None - try: - return stream_info['sources']['lbry_sd_hash'] - except KeyError: - return stream_info.get('stream_hash') - - -def get_output_callback(params): - def callback(l): - return { - 'stream_hash': params.sd_hash if params.stream_info else l.sd_hash, - 'path': os.path.join(params.download_directory, l.file_name) - } - return callback - - -class _DownloadNameHelper(object): - def __init__(self, daemon, name, timeout=DEFAULT_TIMEOUT, download_directory=None, - file_name=None, wait_for_write=True): - self.daemon = daemon - self.name = name - self.timeout = timeout - if not download_directory or not os.path.isdir(download_directory): - self.download_directory = daemon.download_directory - else: - self.download_directory = download_directory - self.file_name = file_name - self.wait_for_write = wait_for_write - - def _setup_stream(self, stream_info): - stream_hash = get_sd_hash(stream_info) - d = self.daemon._get_lbry_file_by_sd_hash(stream_hash) - d.addCallback(self._add_results_callback(stream_info)) - return d - - def _add_results_callback(self, stream_info): - def add_results(l): - if l: - if os.path.isfile(os.path.join(self.download_directory, l.file_name)): - return defer.succeed((stream_info, l)) - return defer.succeed((stream_info, None)) - return add_results - - def wait_or_get_stream(self, args): - stream_info, lbry_file = args - if lbry_file: - return self._get_stream(stream_info) - else: - return self._wait_on_lbry_file(lbry_file) - - def _get_stream(self, stream_info): - d = self.daemon.add_stream( - self.name, self.timeout, self.download_directory, self.file_name, stream_info) - if self.wait_for_write: - d.addCallback(lambda _: self._wait_for_write()) - d.addCallback(lambda _: self.daemon.streams[self.name].downloader) - return d - - def _wait_for_write(self): - file_name = self.daemon.streams[self.name].downloader.file_name - written_bytes = self.get_written_bytes(file_name) - d = defer.succeed(None) - if not written_bytes: - d.addCallback(lambda _: reactor.callLater(1, self._wait_for_write)) - return d - - def _wait_on_lbry_file(self, f): - if f is not None: - written_bytes = self.get_written_bytes(f.file_name) - if written_bytes: - return defer.succeed(self._disp_file(f)) - - d = defer.succeed(None) - d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f)) - return d - - - def get_written_bytes(self, file_name): - try: - file_path = os.path.join(self.download_directory, file_name) - if os.path.isfile(file_path): - written_file = file(file_path) - written_file.seek(0, os.SEEK_END) - written_bytes = written_file.tell() - written_file.close() - else: - written_bytes = False - except Exception: - writen_bytes = False - return written_bytes - - def _disp_file(self, f): - file_path = os.path.join(self.download_directory, f.file_name) - log.info("[%s] Already downloaded: %s --> %s", datetime.now(), f.sd_hash, file_path) - return f - - def _remove_from_wait(self, r): - del self.daemon.waiting_on[self.name] - return r - -class _ResolveNameHelper(object): - def __init__(self, daemon, name, force_refresh): - self.daemon = daemon - self.name = name - self.force_refresh = force_refresh - - def get_deferred(self): - if self.need_fresh_stream(): - log.info("Resolving stream info for lbry://%s", self.name) - d = self.wallet.get_stream_info_for_name(self.name) - d.addCallbacks(self._cache_stream_info, lambda _: defer.fail(UnknownNameError)) - else: - log.info("Returning cached stream info for lbry://%s", self.name) - d = defer.succeed(self.name_data['claim_metadata']) - return d - - @property - def name_data(self): - return self.daemon.name_cache[self.name] - - @property - def wallet(self): - return self.daemon.session.wallet - - def now(self): - return self.daemon._get_long_count_timestamp() - - def _add_txid(self, txid): - self.name_data['txid'] = txid - return defer.succeed(None) - - def _cache_stream_info(self, stream_info): - self.daemon.name_cache[self.name] = { - 'claim_metadata': stream_info, - 'timestamp': self.now() - } - d = self.wallet.get_txid_for_name(self.name) - d.addCallback(self._add_txid) - d.addCallback(lambda _: self.daemon._update_claim_cache()) - d.addCallback(lambda _: self.name_data['claim_metadata']) - return d - - def need_fresh_stream(self): - return self.force_refresh or not self.is_in_cache() or self.is_cached_name_expired() - - def is_in_cache(self): - return self.name in self.daemon.name_cache - - def is_cached_name_expired(self): - time_in_cache = self.now() - self.name_data['timestamp'] - return time_in_cache >= self.daemon.cache_time From f999073fb401c5bb1e1240c9d4e3178c61d7b0b4 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:34:02 -0400 Subject: [PATCH 30/36] revert download refactor to be re-merged after fixes --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 385 +++++++++++---------------- 1 file changed, 160 insertions(+), 225 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 888406e09..b3b234736 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -143,11 +143,6 @@ OK_CODE = 200 REMOTE_SERVER = "www.google.com" -class Parameters(object): - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - class LBRYDaemon(jsonrpc.JSONRPC): """ LBRYnet daemon, a jsonrpc interface to lbry functions @@ -1023,37 +1018,97 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None, - file_name=None, stream_info=None, wait_for_write=True): + file_name=None, stream_info=None, wait_for_write=True): """ Add a lbry file to the file manager, start the download, and return the new lbry file. If it already exists in the file manager, return the existing lbry file """ - helper = _DownloadNameHelper( - self, name, timeout, download_directory, file_name, wait_for_write) + + if not download_directory: + download_directory = self.download_directory + elif not os.path.isdir(download_directory): + download_directory = self.download_directory + + def _remove_from_wait(r): + del self.waiting_on[name] + return r + + def _setup_stream(stream_info): + if 'sources' in stream_info.keys(): + stream_hash = stream_info['sources']['lbry_sd_hash'] + else: + stream_hash = stream_info['stream_hash'] + + d = self._get_lbry_file_by_sd_hash(stream_hash) + def _add_results(l): + if l: + if os.path.isfile(os.path.join(self.download_directory, l.file_name)): + return defer.succeed((stream_info, l)) + return defer.succeed((stream_info, None)) + d.addCallback(_add_results) + return d + + def _wait_on_lbry_file(f): + if os.path.isfile(os.path.join(self.download_directory, f.file_name)): + written_file = file(os.path.join(self.download_directory, f.file_name)) + written_file.seek(0, os.SEEK_END) + written_bytes = written_file.tell() + written_file.close() + else: + written_bytes = False + + if not written_bytes: + d = defer.succeed(None) + d.addCallback(lambda _: reactor.callLater(1, _wait_on_lbry_file, f)) + return d + else: + return defer.succeed(_disp_file(f)) + + def _disp_file(f): + file_path = os.path.join(self.download_directory, f.file_name) + log.info("[" + str(datetime.now()) + "] Already downloaded: " + str(f.sd_hash) + " --> " + file_path) + return f + + def _get_stream(stream_info): + def _wait_for_write(): + try: + if os.path.isfile(os.path.join(self.download_directory, self.streams[name].downloader.file_name)): + written_file = file(os.path.join(self.download_directory, self.streams[name].downloader.file_name)) + written_file.seek(0, os.SEEK_END) + written_bytes = written_file.tell() + written_file.close() + else: + written_bytes = False + except: + written_bytes = False + + if not written_bytes: + d = defer.succeed(None) + d.addCallback(lambda _: reactor.callLater(1, _wait_for_write)) + return d + else: + return defer.succeed(None) + + self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet, + self.lbry_file_manager, max_key_fee=self.max_key_fee, + data_rate=self.data_rate, timeout=timeout, + download_directory=download_directory, file_name=file_name) + d = self.streams[name].start(stream_info, name) + if wait_for_write: + d.addCallback(lambda _: _wait_for_write()) + d.addCallback(lambda _: self.streams[name].downloader) + + return d if not stream_info: self.waiting_on[name] = True d = self._resolve_name(name) else: d = defer.succeed(stream_info) - d.addCallback(helper._setup_stream) - d.addCallback(helper.wait_or_get_stream) + d.addCallback(_setup_stream) + d.addCallback(lambda (stream_info, lbry_file): _get_stream(stream_info) if not lbry_file else _wait_on_lbry_file(lbry_file)) if not stream_info: - d.addCallback(helper._remove_from_wait) - return d - - def add_stream(self, name, timeout, download_directory, file_name, stream_info): - """Makes, adds and starts a stream""" - self.streams[name] = GetStream(self.sd_identifier, - self.session, - self.session.wallet, - self.lbry_file_manager, - max_key_fee=self.max_key_fee, - data_rate=self.data_rate, - timeout=timeout, - download_directory=download_directory, - file_name=file_name) - d = self.streams[name].start(stream_info, name) + d.addCallback(_remove_from_wait) return d def _get_long_count_timestamp(self): @@ -1066,16 +1121,38 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _resolve_name(self, name, force_refresh=False): - """Resolves a name. Checks the cache first before going out to the blockchain. + def _cache_stream_info(stream_info): + def _add_txid(txid): + self.name_cache[name]['txid'] = txid + return defer.succeed(None) - Args: - name: the lbry:// to resolve - force_refresh: if True, always go out to the blockchain to resolve. - """ - if name.startswith('lbry://'): - raise ValueError('name %s should not start with lbry://') - helper = _ResolveNameHelper(self, name, force_refresh) - return helper.get_deferred() + self.name_cache[name] = {'claim_metadata': stream_info, 'timestamp': self._get_long_count_timestamp()} + d = self.session.wallet.get_txid_for_name(name) + d.addCallback(_add_txid) + d.addCallback(lambda _: self._update_claim_cache()) + d.addCallback(lambda _: self.name_cache[name]['claim_metadata']) + + return d + + if not force_refresh: + if name in self.name_cache.keys(): + if (self._get_long_count_timestamp() - self.name_cache[name]['timestamp']) < self.cache_time: + log.info("[" + str(datetime.now()) + "] Returning cached stream info for lbry://" + name) + d = defer.succeed(self.name_cache[name]['claim_metadata']) + else: + log.info("[" + str(datetime.now()) + "] Refreshing stream info for lbry://" + name) + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) + else: + log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) + else: + log.info("[" + str(datetime.now()) + "] Resolving stream info for lbry://" + name) + d = self.session.wallet.get_stream_info_for_name(name) + d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError)) + + return d def _delete_lbry_file(self, lbry_file, delete_file=True): d = self.lbry_file_manager.delete_lbry_file(lbry_file) @@ -1598,54 +1675,65 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d - def _process_get_parameters(self, p): - """Extract info from input parameters and fill in default values for `get` call.""" - # TODO: this process can be abstracted s.t. each method - # can spec what parameters it expects and how to set default values - timeout = p.get('timeout', self.download_timeout) - download_directory = p.get('download_directory', self.download_directory) - file_name = p.get('file_name') - stream_info = p.get('stream_info') - sd_hash = get_sd_hash(stream_info) - wait_for_write = p.get('wait_for_write', True) - name = p.get('name') - return Parameters( - timout=timeout, - download_directory=download_directory, - file_name=file_name, - stream_info=stream_info, - sd_hash=sd_hash, - wait_for_write=wait_for_write, - name=name - ) - def jsonrpc_get(self, p): - """Download stream from a LBRY uri. + """ + Download stream from a LBRY uri Args: 'name': name to download, string 'download_directory': optional, path to directory where file will be saved, string 'file_name': optional, a user specified name for the downloaded file 'stream_info': optional, specified stream info overrides name - 'timout': optional - 'wait_for_write': optional, defaults to True Returns: 'stream_hash': hex string 'path': path of download """ - params = self._process_get_parameters(p) - if not params.name: - return server.failure - if params.name in self.waiting_on: - return server.failure - d = self._download_name(name=params.name, - timeout=params.timeout, - download_directory=params.download_directory, - stream_info=params.stream_info, - file_name=params.file_name, - wait_for_write=params.wait_for_write) - d.addCallback(get_output_callback(params)) - d.addCallback(lambda message: self._render_response(message, OK_CODE)) + + if 'timeout' not in p.keys(): + timeout = self.download_timeout + else: + timeout = p['timeout'] + + if 'download_directory' not in p.keys(): + download_directory = self.download_directory + else: + download_directory = p['download_directory'] + + if 'file_name' in p.keys(): + file_name = p['file_name'] + else: + file_name = None + + if 'stream_info' in p.keys(): + stream_info = p['stream_info'] + if 'sources' in stream_info.keys(): + sd_hash = stream_info['sources']['lbry_sd_hash'] + else: + sd_hash = stream_info['stream_hash'] + else: + stream_info = None + + if 'wait_for_write' in p.keys(): + wait_for_write = p['wait_for_write'] + else: + wait_for_write = True + + if 'name' in p.keys(): + name = p['name'] + if p['name'] not in self.waiting_on.keys(): + d = self._download_name(name=name, timeout=timeout, download_directory=download_directory, + stream_info=stream_info, file_name=file_name, wait_for_write=wait_for_write) + d.addCallback(lambda l: {'stream_hash': sd_hash, + 'path': os.path.join(self.download_directory, l.file_name)} + if stream_info else + {'stream_hash': l.sd_hash, + 'path': os.path.join(self.download_directory, l.file_name)}) + d.addCallback(lambda message: self._render_response(message, OK_CODE)) + else: + d = server.failure + else: + d = server.failure + return d def jsonrpc_stop_lbry_file(self, p): @@ -2192,161 +2280,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = threads.deferToThread(subprocess.Popen, ['open', '-R', path]) else: # No easy way to reveal specific files on Linux, so just open the containing directory - d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.path.dirname(path)]) + d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.dirname(path)]) d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - - -def get_sd_hash(stream_info): - if not stream_info: - return None - try: - return stream_info['sources']['lbry_sd_hash'] - except KeyError: - return stream_info.get('stream_hash') - - -def get_output_callback(params): - def callback(l): - return { - 'stream_hash': params.sd_hash if params.stream_info else l.sd_hash, - 'path': os.path.join(params.download_directory, l.file_name) - } - return callback - - -class _DownloadNameHelper(object): - def __init__(self, daemon, name, timeout=DEFAULT_TIMEOUT, download_directory=None, - file_name=None, wait_for_write=True): - self.daemon = daemon - self.name = name - self.timeout = timeout - if not download_directory or not os.path.isdir(download_directory): - self.download_directory = daemon.download_directory - else: - self.download_directory = download_directory - self.file_name = file_name - self.wait_for_write = wait_for_write - - def _setup_stream(self, stream_info): - stream_hash = get_sd_hash(stream_info) - d = self.daemon._get_lbry_file_by_sd_hash(stream_hash) - d.addCallback(self._add_results_callback(stream_info)) - return d - - def _add_results_callback(self, stream_info): - def add_results(l): - if l: - if os.path.isfile(os.path.join(self.download_directory, l.file_name)): - return defer.succeed((stream_info, l)) - return defer.succeed((stream_info, None)) - return add_results - - def wait_or_get_stream(self, args): - stream_info, lbry_file = args - if lbry_file: - return self._get_stream(stream_info) - else: - return self._wait_on_lbry_file(lbry_file) - - def _get_stream(self, stream_info): - d = self.daemon.add_stream( - self.name, self.timeout, self.download_directory, self.file_name, stream_info) - if self.wait_for_write: - d.addCallback(lambda _: self._wait_for_write()) - d.addCallback(lambda _: self.daemon.streams[self.name].downloader) - return d - - def _wait_for_write(self): - file_name = self.daemon.streams[self.name].downloader.file_name - written_bytes = self.get_written_bytes(file_name) - d = defer.succeed(None) - if not written_bytes: - d.addCallback(lambda _: reactor.callLater(1, self._wait_for_write)) - return d - - def _wait_on_lbry_file(self, f): - written_bytes = self.get_written_bytes(f.file_name) - if not written_bytes: - d = defer.succeed(None) - d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f)) - return d - else: - return defer.succeed(self._disp_file(f)) - - def get_written_bytes(self, file_name): - try: - file_path = os.path.join(self.download_directory, file_name) - if os.path.isfile(file_path): - written_file = file(file_path) - written_file.seek(0, os.SEEK_END) - written_bytes = written_file.tell() - written_file.close() - else: - written_bytes = False - except Exception: - writen_bytes = False - return written_bytes - - def _disp_file(self, f): - file_path = os.path.join(self.download_directory, f.file_name) - log.info("Already downloaded: %s --> %s", f.sd_hash, file_path) - return f - - def _remove_from_wait(self, r): - del self.daemon.waiting_on[self.name] - return r - -class _ResolveNameHelper(object): - def __init__(self, daemon, name, force_refresh): - self.daemon = daemon - self.name = name - self.force_refresh = force_refresh - - def get_deferred(self): - if self.need_fresh_stream(): - log.info("Resolving stream info for lbry://%s", self.name) - d = self.wallet.get_stream_info_for_name(self.name) - d.addCallbacks(self._cache_stream_info, lambda _: defer.fail(UnknownNameError)) - else: - log.info("Returning cached stream info for lbry://%s", self.name) - d = defer.succeed(self.name_data['claim_metadata']) - return d - - @property - def name_data(self): - return self.daemon.name_cache[self.name] - - @property - def wallet(self): - return self.daemon.session.wallet - - def now(self): - return self.daemon._get_long_count_timestamp() - - def _add_txid(self, txid): - self.name_data['txid'] = txid - return defer.succeed(None) - - def _cache_stream_info(self, stream_info): - self.daemon.name_cache[self.name] = { - 'claim_metadata': stream_info, - 'timestamp': self.now() - } - d = self.wallet.get_txid_for_name(self.name) - d.addCallback(self._add_txid) - d.addCallback(lambda _: self.daemon._update_claim_cache()) - d.addCallback(lambda _: self.name_data['claim_metadata']) - return d - - def need_fresh_stream(self): - return self.force_refresh or not self.is_in_cache() or self.is_cached_name_expired() - - def is_in_cache(self): - return self.name in self.daemon.name_cache - - def is_cached_name_expired(self): - time_in_cache = self.now() - self.name_data['timestamp'] - return time_in_cache >= self.daemon.cache_time From 1d31aa9da310fbd9fea4038900786742843fbd6f Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:34:41 -0400 Subject: [PATCH 31/36] Bump version: 0.3.8 -> 0.3.9 --- .bumpversion.cfg | 2 +- lbrynet/__init__.py | 2 +- packaging/ubuntu/lbry.desktop | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b34ee8dd9..cfe8c4436 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.8 +current_version = 0.3.9 commit = True tag = True message = Bump version: {current_version} -> {new_version} diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 49bd262cf..c84517169 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -4,5 +4,5 @@ log = logging.getLogger(__name__) logging.getLogger(__name__).addHandler(logging.NullHandler()) log.setLevel(logging.ERROR) -__version__ = "0.3.8" +__version__ = "0.3.9" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index 1c28d3553..212c67eb9 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.3.8 +Version=0.3.9 Name=LBRY Comment=The world's first user-owned content marketplace Icon=lbry From a22dfb38646fce16157cd5670d400916d7c1e770 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:56:56 -0400 Subject: [PATCH 32/36] cleaner logging from https://github.com/lbryio/lbry/pull/94 --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 51 +++++++++++---------- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 16 ++++++- lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 10 ++-- lbrynet/lbrynet_daemon/LBRYDownloader.py | 2 +- lbrynet/lbrynet_daemon/LBRYPublisher.py | 4 +- 5 files changed, 49 insertions(+), 34 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index aaf504080..b3b234736 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -66,6 +66,11 @@ if not os.path.isdir(log_dir): lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) log = logging.getLogger(__name__) + +# TODO: configuring a logger on module import drastically reduces the +# amount of control the caller of this code has over logging +# +# Better would be to configure all logging at runtime. handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) log.addHandler(handler) log.setLevel(logging.INFO) @@ -461,7 +466,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _announce(): self.announced_startup = True self.startup_status = STARTUP_STAGES[5] - log.info("[" + str(datetime.now()) + "] Started lbrynet-daemon") + log.info("Started lbrynet-daemon") if len(self.startup_scripts): log.info("Scheduling scripts") reactor.callLater(3, self._run_scripts) @@ -483,7 +488,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: _announce()) return d - log.info("[" + str(datetime.now()) + "] Starting lbrynet-daemon") + log.info("Starting lbrynet-daemon") self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) @@ -548,14 +553,14 @@ class LBRYDaemon(jsonrpc.JSONRPC): s = socket.create_connection((host, 80), 2) self.connected_to_internet = True except: - log.info("[" + str(datetime.now()) + "] Internet connection not working") + log.info("Internet connection not working") self.connected_to_internet = False def _check_lbrynet_connection(self): def _log_success(): - log.info("[" + str(datetime.now()) + "] lbrynet connectivity test passed") + log.info("lbrynet connectivity test passed") def _log_failure(): - log.info("[" + str(datetime.now()) + "] lbrynet connectivity test failed") + log.info("lbrynet connectivity test failed") wonderfullife_sh = "6f3af0fa3924be98a54766aa2715d22c6c1509c3f7fa32566df4899a41f3530a9f97b2ecb817fa1dcbf1b30553aefaa7" d = download_sd_blob(self.session, wonderfullife_sh, self.session.base_payment_rate_manager) @@ -573,7 +578,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbryum_version = version return defer.succeed(None) except: - log.info("[" + str(datetime.now()) + "] Failed to get lbryum version from git") + log.info("Failed to get lbryum version from git") self.git_lbryum_version = None return defer.fail(None) @@ -588,7 +593,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.git_lbrynet_version = vr return defer.succeed(None) except: - log.info("[" + str(datetime.now()) + "] Failed to get lbrynet version from git") + log.info("Failed to get lbrynet version from git") self.git_lbrynet_version = None return defer.fail(None) @@ -1168,15 +1173,15 @@ class LBRYDaemon(jsonrpc.JSONRPC): return d d.addCallback(lambda _: finish_deletion(lbry_file)) - d.addCallback(lambda _: log.info("[" + str(datetime.now()) + "] Delete lbry file")) + d.addCallback(lambda _: log.info("Delete lbry file")) return d def _get_est_cost(self, name): def _check_est(d, name): if isinstance(d.result, float): - log.info("[" + str(datetime.now()) + "] Cost est for lbry://" + name + ": " + str(d.result) + "LBC") + log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") else: - log.info("[" + str(datetime.now()) + "] Timeout estimating cost for lbry://" + name + ", using key fee") + log.info("Timeout estimating cost for lbry://" + name + ", using key fee") d.cancel() return defer.succeed(None) @@ -1371,7 +1376,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Returns: true if daemon completed startup, otherwise false """ - log.info("[" + str(datetime.now()) + "] is_running: " + str(self.announced_startup)) + log.info("is_running: " + str(self.announced_startup)) if self.announced_startup: return self._render_response(True, OK_CODE) @@ -1409,7 +1414,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): else: r['message'] = "Catching up with the blockchain" r['progress'] = 0 - log.info("[" + str(datetime.now()) + "] daemon status: " + str(r)) + log.info("daemon status: " + str(r)) return self._render_response(r, OK_CODE) def jsonrpc_is_first_run(self): @@ -1422,7 +1427,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): True if first run, otherwise False """ - log.info("[" + str(datetime.now()) + "] Check if is first run") + log.info("Check if is first run") try: d = self.session.wallet.is_first_run() except: @@ -1442,7 +1447,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): Startup message, such as first run notification """ - log.info("[" + str(datetime.now()) + "] Get startup notice") + log.info("Get startup notice") if self.first_run and not self.session.wallet.wallet_balance: return self._render_response(self.startup_message, OK_CODE) @@ -1482,7 +1487,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'lbryum_update_available': lbryum_version < self.git_lbryum_version } - log.info("[" + str(datetime.now()) + "] Get version info: " + json.dumps(msg)) + log.info("Get version info: " + json.dumps(msg)) return self._render_response(msg, OK_CODE) def jsonrpc_get_settings(self): @@ -1510,7 +1515,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'start_lbrycrdd': bool, """ - log.info("[" + str(datetime.now()) + "] Get daemon settings") + log.info("Get daemon settings") return self._render_response(self.session_settings, OK_CODE) def jsonrpc_set_settings(self, p): @@ -1531,7 +1536,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _log_settings_change(): - log.info("[" + str(datetime.now()) + "] Set daemon settings to " + json.dumps(self.session_settings)) + log.info("Set daemon settings to " + json.dumps(self.session_settings)) d = self._update_settings(p) d.addErrback(lambda err: log.info(err.getTraceback())) @@ -1574,7 +1579,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): balance, float """ - log.info("[" + str(datetime.now()) + "] Get balance") + log.info("Get balance") return self._render_response(float(self.session.wallet.wallet_balance), OK_CODE) def jsonrpc_stop(self): @@ -1815,7 +1820,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.DeferredList(ds) def _disp(results): - log.info('[' + str(datetime.now()) + '] Found ' + str(len(results)) + ' search results') + log.info('Found ' + str(len(results)) + ' search results') consolidated_results = [] for r in results: t = {} @@ -1829,7 +1834,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return consolidated_results - log.info('[' + str(datetime.now()) + '] Search nametrie: ' + search) + log.info('Search nametrie: ' + search) d = self.session.wallet.get_nametrie() d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) @@ -1917,7 +1922,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return server.failure def _disp(x): - log.info("[" + str(datetime.now()) + "] Abandoned name claim tx " + str(x)) + log.info("Abandoned name claim tx " + str(x)) return self._render_response(x, OK_CODE) d = defer.Deferred() @@ -2028,7 +2033,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): """ def _disp(address): - log.info("[" + str(datetime.now()) + "] Got new wallet address: " + address) + log.info("Got new wallet address: " + address) return defer.succeed(address) d = self.session.wallet.get_new_address() @@ -2220,7 +2225,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): exclude_previous = True if 'message' in p.keys(): - log.info("[" + str(datetime.now()) + "] Upload log message: " + str(p['message'])) + log.info("Upload log message: " + str(p['message'])) if 'force' in p.keys(): force = p['force'] diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 309570873..cdde77c6b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -25,8 +25,13 @@ if not os.path.isdir(log_dir): os.mkdir(log_dir) lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) + +DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s" +DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT) + log = logging.getLogger(__name__) handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, backupCount=5) +handler.setFormatter(DEFAULT_FORMATTER) log.addHandler(handler) log.setLevel(logging.INFO) @@ -57,6 +62,13 @@ def stop(): d.callback(None) +def configureConsoleLogger(): + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(DEFAULT_FORMATTER) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(level=logging.INFO) + + def start(): parser = argparse.ArgumentParser(description="Launch lbrynet-daemon") parser.add_argument("--wallet", @@ -75,7 +87,7 @@ def start(): args = parser.parse_args() if args.logtoconsole: - logging.basicConfig(level=logging.INFO) + configureConsoleLogger() args = parser.parse_args() @@ -121,4 +133,4 @@ def start(): return if __name__ == "__main__": - start() \ No newline at end of file + start() diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index 74906da0b..507eb025d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -79,7 +79,7 @@ class LBRYFileStreamer(object): def pauseProducing(self): self._paused = True - log.info("[" + str(datetime.now()) + "] Pausing producer") + log.info("Pausing producer") return defer.succeed(None) def resumeProducing(self): @@ -104,7 +104,7 @@ class LBRYFileStreamer(object): self._request.write(data) self._cursor += 1 - log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % + log.info("Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" % (start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth)) self._sent_bytes = True @@ -117,12 +117,12 @@ class LBRYFileStreamer(object): self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data)) return defer.succeed(None) - log.info("[" + str(datetime.now()) + "] Resuming producer") + log.info("Resuming producer") self._paused = False self._deferred.addCallback(lambda _: _check_for_new_data()) def stopProducing(self): - log.info("[" + str(datetime.now()) + "] Stopping producer") + log.info("Stopping producer") self._stopped = True # self._fileObject.close() self._deferred.addErrback(lambda err: err.trap(defer.CancelledError)) @@ -147,7 +147,7 @@ class HostedLBRYFile(resource.Resource): # # range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-') # start, stop = int(range_header[0]), range_header[1] - # log.info("[" + str(datetime.now()) + "] GET range %s-%s" % (start, stop)) + # log.info("GET range %s-%s" % (start, stop)) # path = os.path.join(self._api.download_directory, stream.file_name) # # d = stream.get_total_bytes() diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index c72444c70..9187cc56d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -147,6 +147,6 @@ class GetStream(object): d = _pay_key_fee() self.downloader = downloader self.download_path = os.path.join(downloader.download_directory, downloader.file_name) - d.addCallback(lambda _: log.info("[%s] Downloading %s --> %s" % (datetime.now(), self.stream_hash, self.downloader.file_name))) + d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name)) d.addCallback(lambda _: self.downloader.start()) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index c0658bfec..bd5c50e4b 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -48,9 +48,7 @@ class Publisher(object): def start(self, name, file_path, bid, metadata, fee=None, sources={}): def _show_result(): - - message = "[%s] Published %s --> lbry://%s txid: %s" % (datetime.now(), self.file_name, self.publish_name, self.txid) - log.info(message) + log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) return defer.succeed(self.txid) self.publish_name = name From 89892042dd75b433e546d070916655b4c68da8f5 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 20:57:05 -0400 Subject: [PATCH 33/36] remove unused line --- lbrynet/core/LBRYWallet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 4a8a4772f..7f82f6e8f 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -1124,7 +1124,6 @@ class LBRYumWallet(LBRYWallet): log.warning("Doing send many. payments to send: %s", str(payments_to_send)) cmd = known_commands['paytomanyandsend'] func = getattr(self.cmd_runner, cmd.name) - # outputs = [(address, amount) for address, amount in payments_to_send.iteritems()] return threads.deferToThread(func, payments_to_send.iteritems()) def _get_value_for_name(self, name): From 4311b218fb95174c48c3bdad87eea37cc317fcf7 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 22:13:22 -0400 Subject: [PATCH 34/36] fix auto_connect --- lbrynet/core/LBRYWallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 7f82f6e8f..99538cea8 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -924,7 +924,7 @@ class LBRYumWallet(LBRYWallet): network_start_d = defer.Deferred() def setup_network(): - self.config = SimpleConfig() + self.config = SimpleConfig({'auto_connect': True}) self.network = Network(self.config) alert.info("Loading the wallet...") return defer.succeed(self.network.start()) From 4888d7a56732dd757f36348f8b43cbf125b4dfab Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 22:34:16 -0400 Subject: [PATCH 35/36] Bump version: 0.3.9 -> 0.3.10 --- .bumpversion.cfg | 2 +- lbrynet/__init__.py | 2 +- packaging/ubuntu/lbry.desktop | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cfe8c4436..715cdfe80 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.9 +current_version = 0.3.10 commit = True tag = True message = Bump version: {current_version} -> {new_version} diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index c84517169..969401685 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -4,5 +4,5 @@ log = logging.getLogger(__name__) logging.getLogger(__name__).addHandler(logging.NullHandler()) log.setLevel(logging.ERROR) -__version__ = "0.3.9" +__version__ = "0.3.10" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index 212c67eb9..7adaf1da7 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.3.9 +Version=0.3.10 Name=LBRY Comment=The world's first user-owned content marketplace Icon=lbry From e67f55bd7447cbee730d895b7f35cf53cf69dba5 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Jul 2016 22:40:55 -0400 Subject: [PATCH 36/36] disable debug logging --- lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py index 0312bf256..6a1fc7c9b 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -54,7 +54,7 @@ class LBRYDaemonRequest(server.Request): memorylimit = 1024*1024*100 # enable/disable debug logging - do_log = True + do_log = False # re-defined only for debug/logging purposes def gotLength(self, length):