From ec4f9011b9714f65891cbe588dbcd966ecc1b195 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 11 Nov 2016 13:40:19 -0500 Subject: [PATCH 1/9] meaningful API error messages --- lbrynet/lbrynet_daemon/DaemonCLI.py | 31 ++++++----- lbrynet/lbrynet_daemon/auth/server.py | 75 ++++++++++++++------------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/lbrynet/lbrynet_daemon/DaemonCLI.py b/lbrynet/lbrynet_daemon/DaemonCLI.py index 2f9fb49ba..8fc32007e 100644 --- a/lbrynet/lbrynet_daemon/DaemonCLI.py +++ b/lbrynet/lbrynet_daemon/DaemonCLI.py @@ -1,15 +1,17 @@ import sys -import json import argparse - +import json from lbrynet.conf import settings from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient +from jsonrpc.common import RPCError -help_msg = "Usage: lbrynet-cli method json-args\n" \ + + +help_msg = "Usage: lbrynet-cli method kwargs\n" \ + "Examples: " \ - + "lbrynet-cli resolve_name '{\"name\": \"what\"}'\n" \ + + "lbrynet-cli resolve_name name=what\n" \ + "lbrynet-cli get_balance\n" \ - + "lbrynet-cli help '{\"function\": \"resolve_name\"}'\n" \ + + "lbrynet-cli help function=resolve_name\n" \ + "\n******lbrynet-cli functions******\n" @@ -59,14 +61,13 @@ def main(): meth = args.method[0] params = {} - if args.params: - if len(args.params) > 1: + if len(args.params) > 1: + params = get_params_from_kwargs(args.params) + elif len(args.params) == 1: + try: + params = json.loads(args.params[0]) + except ValueError: params = get_params_from_kwargs(args.params) - elif len(args.params) == 1: - try: - params = json.loads(args.params[0]) - except ValueError: - params = get_params_from_kwargs(args.params) msg = help_msg for f in api.help(): @@ -83,14 +84,16 @@ def main(): else: result = LBRYAPIClient.config(service=meth, params=params) print json.dumps(result, sort_keys=True) - except: + except RPCError as err: # TODO: The api should return proper error codes # and messages so that they can be passed along to the user # instead of this generic message. # https://app.asana.com/0/158602294500137/200173944358192 - print "Something went wrong, here's the usage for %s:" % meth print api.help({'function': meth}) + print "Here's the traceback for the error you encountered:" + print err.msg + else: print "Unknown function" print msg diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 4a3637fe6..4d5200cd2 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -1,9 +1,10 @@ import logging - from decimal import Decimal from zope.interface import implements from twisted.web import server, resource from twisted.internet import defer +from twisted.python.failure import Failure + from txjsonrpc import jsonrpclib from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError @@ -19,6 +20,12 @@ def default_decimal(obj): return float(obj) +class JSONRPCException(Exception): + def __init__(self, err, code): + self.faultCode = code + self.faultString = err.getTraceback() + + class AuthorizedBase(object): def __init__(self): self.authorized_functions = [] @@ -90,7 +97,16 @@ class AuthJSONRPCServer(AuthorizedBase): def setup(self): return NotImplementedError() + def _render_error(self, request, failure, version=jsonrpclib.VERSION_1, response_code=FAILURE): + fault = jsonrpclib.dumps(JSONRPCException(Failure(failure), response_code), version=version) + self._set_headers(request, fault) + if response_code != AuthJSONRPCServer.FAILURE: + request.setResponseCode(response_code) + request.write(fault) + request.finish() + def render(self, request): + notify_finish = request.notifyFinish() assert self._check_headers(request), InvalidHeaderError session = request.getSession() @@ -114,8 +130,10 @@ class AuthJSONRPCServer(AuthorizedBase): content = request.content.read() try: parsed = jsonrpclib.loads(content) - except ValueError: - return server.failure + except ValueError as err: + log.error("Unable to decode request json") + self._render_error(request, err) + return server.NOT_DONE_YET function_name = parsed.get('method') args = parsed.get('params') @@ -125,36 +143,35 @@ class AuthJSONRPCServer(AuthorizedBase): try: self._run_subhandlers(request) - except SubhandlerError: - return server.failure + except SubhandlerError as err: + self._render_error(request, err, version) + return server.NOT_DONE_YET reply_with_next_secret = False if self._use_authentication: if function_name in self.authorized_functions: try: self._verify_token(session_id, parsed, token) - except InvalidAuthenticationToken: - log.warning("API validation failed") - request.setResponseCode(self.UNAUTHORIZED) - request.finish() + except InvalidAuthenticationToken as err: + log.error("API validation failed") + self._render_error(request, err, version, response_code=AuthJSONRPCServer.UNAUTHORIZED) return server.NOT_DONE_YET self._update_session_secret(session_id) reply_with_next_secret = True try: function = self._get_jsonrpc_method(function_name) - except Exception: - log.warning("Unknown method: %s", function_name) - return server.failure + except AttributeError as err: + log.error("Unknown method: %s", function_name) + self._render_error(request, err, version) + return server.NOT_DONE_YET d = defer.maybeDeferred(function) if args == [{}] else defer.maybeDeferred(function, *args) - # cancel the response if the connection is broken - notify_finish = request.notifyFinish() - notify_finish.addErrback(self._response_failed, d) - d.addErrback(self._errback_render, id) - d.addCallback(self._callback_render, request, id, version, reply_with_next_secret) - d.addErrback(notify_finish.errback) + # cancel the response if the connection is broken + notify_finish.addErrback(self._response_failed, d) + d.addCallback(self._callback_render, request, version, reply_with_next_secret) + d.addErrback(lambda err: self._render_error(request, err, version)) return server.NOT_DONE_YET def _register_user_session(self, session_id): @@ -209,7 +226,7 @@ class AuthJSONRPCServer(AuthorizedBase): return True def _get_jsonrpc_method(self, function_path): - assert self._check_function_path(function_path) + assert self._check_function_path(function_path), AttributeError(function_path) return self.callable_methods.get(function_path) def _initialize_session(self, session_id): @@ -242,9 +259,9 @@ class AuthJSONRPCServer(AuthorizedBase): assert handler(request) except Exception as err: log.error(err.message) - raise SubhandlerError + raise SubhandlerError(err.message) - def _callback_render(self, result, request, id, version, auth_required=False): + def _callback_render(self, result, request, version, auth_required=False): result_for_return = result if not isinstance(result, dict) else result['result'] if version == jsonrpclib.VERSION_PRE1: @@ -255,20 +272,8 @@ class AuthJSONRPCServer(AuthorizedBase): encoded_message = jsonrpclib.dumps(result_for_return, version=version, default=default_decimal) self._set_headers(request, encoded_message, auth_required) self._render_message(request, encoded_message) - except: - fault = jsonrpclib.Fault(self.FAILURE, "can't serialize output") - encoded_message = jsonrpclib.dumps(fault, version=version) - self._set_headers(request, encoded_message) - self._render_message(request, encoded_message) - - def _errback_render(self, failure, id): - log.error("Request failed:") - log.error(failure) - log.error(failure.value) - log.error(id) - if isinstance(failure.value, jsonrpclib.Fault): - return failure.value - return server.failure + except Exception as err: + self._render_error(request, err, response_code=self.FAILURE, version=version) def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) From f9fef3d586b80b183af90329fe5dedcb30f5a480 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 11 Nov 2016 13:41:36 -0500 Subject: [PATCH 2/9] reactor.iterate() in dht shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -fixes enigmatic `’Port' object has no attribute 'socket’` error --- lbrynet/dht/protocol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 37c855c50..8f1d0fd87 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -321,3 +321,4 @@ class KademliaProtocol(protocol.DatagramProtocol): except Exception, e: log.exception('Failed to cancel %s', self._callLaterList[key]) del self._callLaterList[key] + reactor.iterate() From 7f96e35b0ef66b1921b13bbc2fb26a604bedefdd Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 11 Nov 2016 13:42:51 -0500 Subject: [PATCH 3/9] clean up daemon imports and settings var name --- lbrynet/lbrynet_daemon/Daemon.py | 157 +++++++++++++------------------ 1 file changed, 67 insertions(+), 90 deletions(-) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index b8b434030..abcfdfb87 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -10,7 +10,6 @@ import sys import base58 import requests import simplejson as json - from urllib2 import urlopen from appdirs import user_data_dir from datetime import datetime @@ -21,55 +20,34 @@ from twisted.internet.task import LoopingCall from txjsonrpc import jsonrpclib from jsonschema import ValidationError -from lbrynet import __version__ as lbrynet_version # TODO: importing this when internet is disabled raises a socket.gaierror from lbryum.version import LBRYUM_VERSION as lbryum_version - from lbrynet import __version__ as lbrynet_version -from lbrynet import conf -from lbrynet.conf import settings as lbrynet_settings -from lbrynet import analytics -from lbrynet import reflector -from lbrynet.metadata.Metadata import Metadata, verify_name_characters +from lbrynet import conf, reflector, analytics from lbrynet.metadata.Fee import FeeValidator -from lbrynet.core import log_support -from lbrynet.core import utils -from lbrynet.core.utils import generate_id -from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader -from lbrynet.core.Session import Session -from lbrynet.core.looping_call_manager import LoopingCallManager -from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory -from lbrynet.core.server.ServerProtocol import ServerProtocolFactory -from lbrynet.core.Error import InsufficientFundsError, InvalidNameError -from lbrynet.core.PTCWallet import PTCWallet -from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet -from lbrynet.lbrynet_console.Settings import Settings -from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager -from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType +from lbrynet.metadata.Metadata import Metadata, verify_name_characters from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileSaverFactory, EncryptedFileOpenerFactory from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager from lbrynet.lbryfile.EncryptedFileMetadataManager import TempEncryptedFileMetadataManager +from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType +from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager +from lbrynet.lbrynet_console.Settings import Settings from lbrynet.lbrynet_daemon.UIManager import UIManager from lbrynet.lbrynet_daemon.Downloader import GetStream from lbrynet.lbrynet_daemon.Publisher import Publisher from lbrynet.lbrynet_daemon.ExchangeRateManager import ExchangeRateManager -from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient from lbrynet.lbrynet_daemon.auth.server import AuthJSONRPCServer - -from lbrynet.metadata.Metadata import Metadata, verify_name_characters from lbrynet.core import log_support from lbrynet.core import utils -from lbrynet.core.utils import generate_id -from lbrynet.lbrynet_console.Settings import Settings - from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader from lbrynet.core.Session import Session from lbrynet.core.PTCWallet import PTCWallet from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet -from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager -from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager, TempEncryptedFileMetadataManager -from lbrynet import reflector +from lbrynet.core.looping_call_manager import LoopingCallManager +from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory +from lbrynet.core.server.ServerProtocol import ServerProtocolFactory +from lbrynet.core.Error import InsufficientFundsError, InvalidNameError log = logging.getLogger(__name__) @@ -222,7 +200,7 @@ class Daemon(AuthJSONRPCServer): """ def __init__(self, root): - AuthJSONRPCServer.__init__(self, lbrynet_settings.use_auth_http) + AuthJSONRPCServer.__init__(self, conf.settings.use_auth_http) reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) self.allowed_during_startup = [ @@ -232,38 +210,38 @@ class Daemon(AuthJSONRPCServer): 'version', 'get_search_servers' ] last_version = {'last_version': {'lbrynet': lbrynet_version, 'lbryum': lbryum_version}} - lbrynet_settings.update(last_version) - self.db_dir = lbrynet_settings.data_dir - self.download_directory = lbrynet_settings.download_directory + conf.settings.update(last_version) + self.db_dir = conf.settings.data_dir + self.download_directory = conf.settings.download_directory self.created_data_dir = False if not os.path.exists(self.db_dir): os.mkdir(self.db_dir) self.created_data_dir = True - if lbrynet_settings.BLOBFILES_DIR == "blobfiles": + if conf.settings.BLOBFILES_DIR == "blobfiles": self.blobfile_dir = os.path.join(self.db_dir, "blobfiles") else: - log.info("Using non-default blobfiles directory: %s", lbrynet_settings.BLOBFILES_DIR) - self.blobfile_dir = lbrynet_settings.BLOBFILES_DIR + log.info("Using non-default blobfiles directory: %s", conf.settings.BLOBFILES_DIR) + self.blobfile_dir = conf.settings.BLOBFILES_DIR - self.run_on_startup = lbrynet_settings.run_on_startup - self.data_rate = lbrynet_settings.data_rate - self.max_key_fee = lbrynet_settings.max_key_fee - self.max_upload = lbrynet_settings.max_upload - self.max_download = lbrynet_settings.max_download - self.upload_log = lbrynet_settings.upload_log - self.search_timeout = lbrynet_settings.search_timeout - self.download_timeout = lbrynet_settings.download_timeout - self.max_search_results = lbrynet_settings.max_search_results - self.run_reflector_server = lbrynet_settings.run_reflector_server - self.wallet_type = lbrynet_settings.wallet - self.delete_blobs_on_remove = lbrynet_settings.delete_blobs_on_remove - self.peer_port = lbrynet_settings.peer_port - self.reflector_port = lbrynet_settings.reflector_port - self.dht_node_port = lbrynet_settings.dht_node_port - self.use_upnp = lbrynet_settings.use_upnp - self.start_lbrycrdd = lbrynet_settings.start_lbrycrdd - self.cache_time = lbrynet_settings.cache_time - self.startup_scripts = lbrynet_settings.startup_scripts + self.run_on_startup = conf.settings.run_on_startup + self.data_rate = conf.settings.data_rate + self.max_key_fee = conf.settings.max_key_fee + self.max_upload = conf.settings.max_upload + self.max_download = conf.settings.max_download + self.upload_log = conf.settings.upload_log + self.search_timeout = conf.settings.search_timeout + self.download_timeout = conf.settings.download_timeout + self.max_search_results = conf.settings.max_search_results + self.run_reflector_server = conf.settings.run_reflector_server + self.wallet_type = conf.settings.wallet + self.delete_blobs_on_remove = conf.settings.delete_blobs_on_remove + self.peer_port = conf.settings.peer_port + self.reflector_port = conf.settings.reflector_port + self.dht_node_port = conf.settings.dht_node_port + self.use_upnp = conf.settings.use_upnp + self.start_lbrycrdd = conf.settings.start_lbrycrdd + self.cache_time = conf.settings.cache_time + self.startup_scripts = conf.settings.startup_scripts self.startup_status = STARTUP_STAGES[0] self.startup_message = None @@ -275,11 +253,11 @@ class Daemon(AuthJSONRPCServer): self.ui_version = None self.ip = None self.first_run = None - self.log_file = lbrynet_settings.get_log_filename() + self.log_file = conf.settings.get_log_filename() self.current_db_revision = 1 self.session = None self.uploaded_temp_files = [] - self._session_id = base58.b58encode(generate_id()) + self._session_id = base58.b58encode(utils.generate_id()) # TODO: this should probably be passed into the daemon, or # possibly have the entire log upload functionality taken out # of the daemon, but I don't want to deal with that now @@ -287,7 +265,7 @@ class Daemon(AuthJSONRPCServer): self.analytics_manager = None self.lbryid = PENDING_LBRY_ID - self.daemon_conf = lbrynet_settings.get_conf_filename() + self.daemon_conf = conf.settings.get_conf_filename() self.wallet_user = None self.wallet_password = None @@ -321,7 +299,6 @@ class Daemon(AuthJSONRPCServer): parsed = jsonrpclib.loads(content) function_path = parsed.get("method") if self.wallet_type == "lbryum" and function_path in ['set_miner', 'get_miner_status']: - log.warning("Mining commands are not available in lbryum") raise Exception("Command not available in lbryum") return True @@ -389,7 +366,7 @@ class Daemon(AuthJSONRPCServer): self.exchange_rate_manager.start() d = defer.Deferred() - if lbrynet_settings.host_ui: + if conf.settings.host_ui: self.lbry_ui_manager.update_checker.start(1800, now=False) d.addCallback(lambda _: self.lbry_ui_manager.setup()) d.addCallback(lambda _: self._initial_setup()) @@ -678,27 +655,27 @@ class Daemon(AuthJSONRPCServer): for key, setting_type in setting_types.iteritems(): if key in settings: if isinstance(settings[key], setting_type): - lbrynet_settings.update({key: settings[key]}) + conf.settings.update({key: settings[key]}) elif key == "max_key_fee" and isinstance(FeeValidator(settings[key]).amount, setting_type): - lbrynet_settings.update({key: settings[key]}) + conf.settings.update({key: settings[key]}) else: try: converted = setting_type(settings[key]) - lbrynet_settings.update({key: converted}) + conf.settings.update({key: converted}) except Exception as err: log.warning(err.message) log.warning("error converting setting '%s' to type %s", key, setting_type) - self.run_on_startup = lbrynet_settings.run_on_startup - self.data_rate = lbrynet_settings.data_rate - self.max_key_fee = lbrynet_settings.max_key_fee - self.download_directory = lbrynet_settings.download_directory - self.max_upload = lbrynet_settings.max_upload - self.max_download = lbrynet_settings.max_download - self.upload_log = lbrynet_settings.upload_log - self.download_timeout = lbrynet_settings.download_timeout - self.search_timeout = lbrynet_settings.search_timeout - self.cache_time = lbrynet_settings.cache_time + self.run_on_startup = conf.settings.run_on_startup + self.data_rate = conf.settings.data_rate + self.max_key_fee = conf.settings.max_key_fee + self.download_directory = conf.settings.download_directory + self.max_upload = conf.settings.max_upload + self.max_download = conf.settings.max_download + self.upload_log = conf.settings.upload_log + self.download_timeout = conf.settings.download_timeout + self.search_timeout = conf.settings.search_timeout + self.cache_time = conf.settings.cache_time return defer.succeed(True) @@ -755,7 +732,7 @@ class Daemon(AuthJSONRPCServer): self.lbryid = lbryid def _make_set_and_save_lbryid(self): - self.lbryid = generate_id() + self.lbryid = utils.generate_id() log.info("Generated new LBRY ID: " + base58.b58encode(self.lbryid)) d = self.settings.save_lbryid(self.lbryid) return d @@ -797,7 +774,7 @@ class Daemon(AuthJSONRPCServer): def get_default_data_rate(): d = self.settings.get_default_data_payment_rate() d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else - lbrynet_settings.data_rate}) + conf.settings.data_rate}) return d def get_wallet(): @@ -811,8 +788,8 @@ class Daemon(AuthJSONRPCServer): elif self.wallet_type == "lbryum": log.info("Using lbryum wallet") config = {'auto-connect': True} - if lbrynet_settings.lbryum_wallet_dir: - config['lbryum_path'] = lbrynet_settings.lbryum_wallet_dir + if conf.settings.lbryum_wallet_dir: + config['lbryum_path'] = conf.settings.lbryum_wallet_dir d = defer.succeed(LBRYumWallet(self.db_dir, config)) elif self.wallet_type == "ptc": log.info("Using PTC wallet") @@ -835,7 +812,7 @@ class Daemon(AuthJSONRPCServer): def create_session(results): self.session = Session(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid, blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port, - known_dht_nodes=lbrynet_settings.known_dht_nodes, peer_port=self.peer_port, + known_dht_nodes=conf.settings.known_dht_nodes, peer_port=self.peer_port, use_upnp=self.use_upnp, wallet=results['wallet']) self.startup_status = STARTUP_STAGES[2] @@ -857,7 +834,7 @@ class Daemon(AuthJSONRPCServer): self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_opener_factory) return defer.succeed(None) - def _download_sd_blob(self, sd_hash, timeout=lbrynet_settings.sd_download_timeout): + def _download_sd_blob(self, sd_hash, timeout=conf.settings.sd_download_timeout): def cb(result): if not r.called: r.callback(result) @@ -875,7 +852,7 @@ class Daemon(AuthJSONRPCServer): return r - def _download_name(self, name, timeout=lbrynet_settings.download_timeout, download_directory=None, + def _download_name(self, name, timeout=conf.settings.download_timeout, download_directory=None, file_name=None, stream_info=None, wait_for_write=True): """ Add a lbry file to the file manager, start the download, and return the new lbry file. @@ -1030,7 +1007,7 @@ class Daemon(AuthJSONRPCServer): log.info("Reflecting stream: %s" % stream_hash) - reflector_server = random.choice(lbrynet_settings.reflector_servers) + reflector_server = random.choice(conf.settings.reflector_servers) reflector_address, reflector_port = reflector_server[0], reflector_server[1] log.info("Start reflector client") factory = reflector.ClientFactory( @@ -1049,7 +1026,7 @@ class Daemon(AuthJSONRPCServer): log.info("Reflecting %i blobs" % len(blob_hashes)) - reflector_server = random.choice(lbrynet_settings.reflector_servers) + reflector_server = random.choice(conf.settings.reflector_servers) reflector_address, reflector_port = reflector_server[0], reflector_server[1] log.info("Start reflector client") factory = reflector.BlobClientFactory( @@ -1072,7 +1049,7 @@ class Daemon(AuthJSONRPCServer): log.info("Removing one time startup scripts") remaining_scripts = [s for s in self.startup_scripts if 'run_once' not in s.keys()] startup_scripts = self.startup_scripts - self.startup_scripts = lbrynet_settings.startup_scripts = remaining_scripts + self.startup_scripts = conf.settings.startup_scripts = remaining_scripts conf.save_settings() for script in startup_scripts: @@ -1251,7 +1228,7 @@ class Daemon(AuthJSONRPCServer): """ log.info("Get daemon settings") - return self._render_response(lbrynet_settings.__dict__, OK_CODE) + return self._render_response(conf.settings.__dict__, OK_CODE) @AuthJSONRPCServer.auth_required def jsonrpc_set_settings(self, p): @@ -1272,12 +1249,12 @@ class Daemon(AuthJSONRPCServer): """ def _log_settings_change(): - log.info("Set daemon settings to " + json.dumps(lbrynet_settings.configurable_settings)) + log.info("Set daemon settings to " + json.dumps(conf.settings.configurable_settings)) d = self._update_settings(p) d.addErrback(lambda err: log.info(err.getTraceback())) d.addCallback(lambda _: _log_settings_change()) - d.addCallback(lambda _: self._render_response(lbrynet_settings.configurable_settings, OK_CODE)) + d.addCallback(lambda _: self._render_response(conf.settings.configurable_settings, OK_CODE)) return d @@ -2036,7 +2013,7 @@ class Daemon(AuthJSONRPCServer): sd blob, dict """ sd_hash = p[FileID.SD_HASH] - timeout = p.get('timeout', lbrynet_settings.sd_download_timeout) + timeout = p.get('timeout', conf.settings.sd_download_timeout) d = self._download_sd_blob(sd_hash, timeout) d.addCallbacks( lambda r: self._render_response(r, OK_CODE), @@ -2401,7 +2378,7 @@ def get_darwin_lbrycrdd_path(): class _DownloadNameHelper(object): - def __init__(self, daemon, name, timeout=lbrynet_settings.download_timeout, download_directory=None, + def __init__(self, daemon, name, timeout=conf.settings.download_timeout, download_directory=None, file_name=None, wait_for_write=True): self.daemon = daemon self.name = name From 472bb6af1b6b51e9b5602afc0086b17bb5a34c32 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 11 Nov 2016 13:43:58 -0500 Subject: [PATCH 4/9] remove unused lighthouse search from daemon --- lbrynet/conf.py | 1 - lbrynet/lbrynet_daemon/Daemon.py | 54 +--------------------------- lbrynet/lbrynet_daemon/Lighthouse.py | 27 -------------- 3 files changed, 1 insertion(+), 81 deletions(-) delete mode 100644 lbrynet/lbrynet_daemon/Lighthouse.py diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 2a12af014..66e273929 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -160,7 +160,6 @@ class AdjustableSettings(Setting): self.check_ui_requirements = True self.local_ui_path = False self.api_port = 5279 - self.search_servers = ['lighthouse1.lbry.io:50005'] self.data_rate = .0001 # points/megabyte self.min_info_rate = .02 # points/1000 infos self.min_valuable_info_rate = .05 # points/1000 infos diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index abcfdfb87..d03b593e4 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -207,7 +207,7 @@ class Daemon(AuthJSONRPCServer): 'is_running', 'is_first_run', 'get_time_behind_blockchain', 'stop', 'daemon_status', 'get_start_notice', - 'version', 'get_search_servers' + 'version' ] last_version = {'last_version': {'lbrynet': lbrynet_version, 'lbryum': lbryum_version}} conf.settings.update(last_version) @@ -276,7 +276,6 @@ class Daemon(AuthJSONRPCServer): self.name_cache = {} self.set_wallet_attributes() self.exchange_rate_manager = ExchangeRateManager() - self.lighthouse_client = LighthouseClient() calls = { Checker.INTERNET_CONNECTION: LoopingCall(CheckInternetConnection(self)), Checker.VERSION: LoopingCall(CheckRemoteVersions(self)), @@ -1065,9 +1064,6 @@ class Daemon(AuthJSONRPCServer): return defer.succeed(None) - def _search(self, search): - return self.lighthouse_client.search(search) - def jsonrpc_is_running(self): """ Check if lbrynet daemon is running @@ -1567,42 +1563,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - def jsonrpc_search_nametrie(self, p): - """ - Search the nametrie for claims - - Args: - 'search': search query, string - Returns: - List of search results - """ - - # TODO: change this function to "search" - - if 'search' in p.keys(): - search = p['search'] - else: - return self._render_response(None, BAD_REQUEST) - - # TODO: have ui accept the actual outputs - def _clean(n): - t = [] - for i in n: - td = {k: i['value'][k] for k in i['value']} - td['cost_est'] = float(i['cost']) - td['thumbnail'] = i['value'].get('thumbnail', "img/Free-speech-flag.svg") - td['name'] = i['name'] - t.append(td) - return t - - log.info('Search: %s' % search) - - d = self._search(search) - d.addCallback(_clean) - d.addCallback(lambda results: self._render_response(results, OK_CODE)) - - return d - @AuthJSONRPCServer.auth_required def jsonrpc_delete_lbry_file(self, p): """ @@ -2248,18 +2208,6 @@ class Daemon(AuthJSONRPCServer): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - def jsonrpc_get_search_servers(self): - """ - Get list of lighthouse servers - Args: - None - Returns: - List of address:port - """ - - d = self._render_response(lbrynet_settings.search_servers, OK_CODE) - return d - def jsonrpc_get_mean_availability(self): """ Get mean blob availability diff --git a/lbrynet/lbrynet_daemon/Lighthouse.py b/lbrynet/lbrynet_daemon/Lighthouse.py deleted file mode 100644 index fb574e55c..000000000 --- a/lbrynet/lbrynet_daemon/Lighthouse.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging -import random -from txjsonrpc.web.jsonrpc import Proxy -from lbrynet.conf import settings - -log = logging.getLogger(__name__) - - -class LighthouseClient(object): - def __init__(self, servers=None): - self.servers = servers or settings.search_servers - - def _get_random_server(self): - return Proxy(random.choice(self.servers)) - - def _run_query(self, func, arg): - return self._get_random_server().callRemote(func, arg) - - def search(self, search): - return self._run_query('search', search) - - def announce_sd(self, sd_hash): - log.info("Announce sd to lighthouse") - return self._run_query('announce_sd', sd_hash) - - def check_available(self, sd_hash): - return self._run_query('check_available', sd_hash) From f45a2cba320539bed17ee747cb7a67ec91574078 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 14 Nov 2016 13:41:05 -0500 Subject: [PATCH 5/9] fix insufficient funds and max key fee bug --- lbrynet/lbrynet_daemon/Downloader.py | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lbrynet/lbrynet_daemon/Downloader.py b/lbrynet/lbrynet_daemon/Downloader.py index 97446f2ef..e041600fb 100644 --- a/lbrynet/lbrynet_daemon/Downloader.py +++ b/lbrynet/lbrynet_daemon/Downloader.py @@ -77,13 +77,10 @@ class GetStream(object): self.finished.callback((False, None, None)) def _convert_max_fee(self): - if isinstance(self.max_key_fee, dict): - max_fee = FeeValidator(self.max_key_fee) - if max_fee.currency_symbol == "LBC": - return max_fee.amount - return self.exchange_rate_manager.to_lbc(self.fee).amount - elif isinstance(self.max_key_fee, float): - return float(self.max_key_fee) + max_fee = FeeValidator(self.max_key_fee) + if max_fee.currency_symbol == "LBC": + return max_fee.amount + return self.exchange_rate_manager.to_lbc(self.max_key_fee).amount def start(self, stream_info, name): def _cause_timeout(err): @@ -117,14 +114,18 @@ class GetStream(object): if 'fee' in self.stream_info: self.fee = FeeValidator(self.stream_info['fee']) max_key_fee = self._convert_max_fee() - if self.exchange_rate_manager.to_lbc(self.fee).amount > max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount, - self.max_key_fee, - self.resolved_name)) + converted_fee = self.exchange_rate_manager.to_lbc(self.fee).amount + if converted_fee > self.wallet.get_balance(): + log.warning("Insufficient funds to download lbry://%s", self.resolved_name) + return defer.fail(InsufficientFundsError()) + if converted_fee > max_key_fee: + log.warning("Key fee %f above limit of %f didn't download lbry://%s", converted_fee, + max_key_fee, + self.resolved_name) return defer.fail(KeyFeeAboveMaxAllowed()) - log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee), - max_key_fee, - self.resolved_name)) + log.info("Key fee %f below limit of %f, downloading lbry://%s", converted_fee, + max_key_fee, + self.resolved_name) self.checker.start(1) From 908c9a3bfa69cc7c77e3022e6c18a1ceed199f38 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 14 Nov 2016 13:53:11 -0500 Subject: [PATCH 6/9] feedback from job --- lbrynet/lbrynet_daemon/auth/server.py | 40 ++++++++++++++++----------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 4d5200cd2..9784ba16d 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -6,7 +6,7 @@ from twisted.internet import defer from twisted.python.failure import Failure from txjsonrpc import jsonrpclib - +from lbrynet.core import log_support from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError from lbrynet.conf import settings from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message @@ -23,7 +23,11 @@ def default_decimal(obj): class JSONRPCException(Exception): def __init__(self, err, code): self.faultCode = code - self.faultString = err.getTraceback() + self.err = err + + @property + def faultString(self): + return self.err.getTraceback() class AuthorizedBase(object): @@ -98,7 +102,8 @@ class AuthJSONRPCServer(AuthorizedBase): return NotImplementedError() def _render_error(self, request, failure, version=jsonrpclib.VERSION_1, response_code=FAILURE): - fault = jsonrpclib.dumps(JSONRPCException(Failure(failure), response_code), version=version) + err = JSONRPCException(Failure(failure), response_code) + fault = jsonrpclib.dumps(err, version=version) self._set_headers(request, fault) if response_code != AuthJSONRPCServer.FAILURE: request.setResponseCode(response_code) @@ -108,12 +113,12 @@ class AuthJSONRPCServer(AuthorizedBase): def render(self, request): notify_finish = request.notifyFinish() assert self._check_headers(request), InvalidHeaderError - session = request.getSession() session_id = session.uid if self._use_authentication: - # if this is a new session, send a new secret and set the expiration, otherwise, session.touch() + # if this is a new session, send a new secret and set the expiration + # otherwise, session.touch() if self._initialize_session(session_id): def expire_session(): self._unregister_user_session(session_id) @@ -131,7 +136,7 @@ class AuthJSONRPCServer(AuthorizedBase): try: parsed = jsonrpclib.loads(content) except ValueError as err: - log.error("Unable to decode request json") + log.warning("Unable to decode request json") self._render_error(request, err) return server.NOT_DONE_YET @@ -153,8 +158,9 @@ class AuthJSONRPCServer(AuthorizedBase): try: self._verify_token(session_id, parsed, token) except InvalidAuthenticationToken as err: - log.error("API validation failed") - self._render_error(request, err, version, response_code=AuthJSONRPCServer.UNAUTHORIZED) + log.warning("API validation failed") + self._render_error(request, err, version, + response_code=AuthJSONRPCServer.UNAUTHORIZED) return server.NOT_DONE_YET self._update_session_secret(session_id) reply_with_next_secret = True @@ -162,11 +168,14 @@ class AuthJSONRPCServer(AuthorizedBase): try: function = self._get_jsonrpc_method(function_name) except AttributeError as err: - log.error("Unknown method: %s", function_name) + log.warning("Unknown method: %s", function_name) self._render_error(request, err, version) return server.NOT_DONE_YET - d = defer.maybeDeferred(function) if args == [{}] else defer.maybeDeferred(function, *args) + if args == [{}]: + d = defer.maybeDeferred(function) + else: + d = defer.maybeDeferred(function, *args) # cancel the response if the connection is broken notify_finish.addErrback(self._response_failed, d) @@ -226,7 +235,8 @@ class AuthJSONRPCServer(AuthorizedBase): return True def _get_jsonrpc_method(self, function_path): - assert self._check_function_path(function_path), AttributeError(function_path) + if not self._check_function_path(function_path): + raise AttributeError(function_path) return self.callable_methods.get(function_path) def _initialize_session(self, session_id): @@ -255,11 +265,8 @@ class AuthJSONRPCServer(AuthorizedBase): def _run_subhandlers(self, request): for handler in self.subhandlers: - try: - assert handler(request) - except Exception as err: - log.error(err.message) - raise SubhandlerError(err.message) + if not handler(request): + raise SubhandlerError("Subhandler error processing request: %s", request) def _callback_render(self, result, request, version, auth_required=False): result_for_return = result if not isinstance(result, dict) else result['result'] @@ -273,6 +280,7 @@ class AuthJSONRPCServer(AuthorizedBase): self._set_headers(request, encoded_message, auth_required) self._render_message(request, encoded_message) except Exception as err: + log.exception(err.message) self._render_error(request, err, response_code=self.FAILURE, version=version) def _render_response(self, result, code): From 8f984699694b26821fec3c1b9eec71879e7a2e9e Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 14 Nov 2016 14:10:19 -0500 Subject: [PATCH 7/9] unused imports --- lbrynet/lbrynet_daemon/Downloader.py | 1 - lbrynet/lbrynet_daemon/auth/server.py | 1 - 2 files changed, 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/Downloader.py b/lbrynet/lbrynet_daemon/Downloader.py index e041600fb..99db3962a 100644 --- a/lbrynet/lbrynet_daemon/Downloader.py +++ b/lbrynet/lbrynet_daemon/Downloader.py @@ -1,4 +1,3 @@ -import json import logging import os diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 9784ba16d..2942dd91f 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -6,7 +6,6 @@ from twisted.internet import defer from twisted.python.failure import Failure from txjsonrpc import jsonrpclib -from lbrynet.core import log_support from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError from lbrynet.conf import settings from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message From ba7f326e7298e9c202afd78d272b0e9977a6a924 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 16 Nov 2016 15:16:15 -0500 Subject: [PATCH 8/9] use log_support --- lbrynet/lbrynet_daemon/auth/server.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 2942dd91f..3ae8d7ff9 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -7,6 +7,7 @@ from twisted.python.failure import Failure from txjsonrpc import jsonrpclib from lbrynet.core.Error import InvalidAuthenticationToken, InvalidHeaderError, SubhandlerError +from lbrynet.core import log_support from lbrynet.conf import settings from lbrynet.lbrynet_daemon.auth.util import APIKey, get_auth_message from lbrynet.lbrynet_daemon.auth.client import LBRY_SECRET @@ -100,7 +101,11 @@ class AuthJSONRPCServer(AuthorizedBase): def setup(self): return NotImplementedError() - def _render_error(self, request, failure, version=jsonrpclib.VERSION_1, response_code=FAILURE): + def _render_error(self, failure, request, version=jsonrpclib.VERSION_1, response_code=FAILURE, + log_failure=False, log_msg=None): + if log_failure: + msg = log_msg or "API Failure: %s" + log_support.failure(Failure(failure), log, msg) err = JSONRPCException(Failure(failure), response_code) fault = jsonrpclib.dumps(err, version=version) self._set_headers(request, fault) @@ -136,7 +141,7 @@ class AuthJSONRPCServer(AuthorizedBase): parsed = jsonrpclib.loads(content) except ValueError as err: log.warning("Unable to decode request json") - self._render_error(request, err) + self._render_error(err, request) return server.NOT_DONE_YET function_name = parsed.get('method') @@ -148,7 +153,7 @@ class AuthJSONRPCServer(AuthorizedBase): try: self._run_subhandlers(request) except SubhandlerError as err: - self._render_error(request, err, version) + self._render_error(err, request, version) return server.NOT_DONE_YET reply_with_next_secret = False @@ -158,7 +163,7 @@ class AuthJSONRPCServer(AuthorizedBase): self._verify_token(session_id, parsed, token) except InvalidAuthenticationToken as err: log.warning("API validation failed") - self._render_error(request, err, version, + self._render_error(err, request, version, response_code=AuthJSONRPCServer.UNAUTHORIZED) return server.NOT_DONE_YET self._update_session_secret(session_id) @@ -168,7 +173,7 @@ class AuthJSONRPCServer(AuthorizedBase): function = self._get_jsonrpc_method(function_name) except AttributeError as err: log.warning("Unknown method: %s", function_name) - self._render_error(request, err, version) + self._render_error(err, request, version) return server.NOT_DONE_YET if args == [{}]: @@ -179,7 +184,7 @@ class AuthJSONRPCServer(AuthorizedBase): # cancel the response if the connection is broken notify_finish.addErrback(self._response_failed, d) d.addCallback(self._callback_render, request, version, reply_with_next_secret) - d.addErrback(lambda err: self._render_error(request, err, version)) + d.addErrback(self._render_error, request, version, log_failure=True) return server.NOT_DONE_YET def _register_user_session(self, session_id): @@ -279,8 +284,9 @@ class AuthJSONRPCServer(AuthorizedBase): self._set_headers(request, encoded_message, auth_required) self._render_message(request, encoded_message) except Exception as err: - log.exception(err.message) - self._render_error(request, err, response_code=self.FAILURE, version=version) + msg = "Failed to render API response: %s" + self._render_error(err, request, response_code=self.FAILURE, version=version, + log_failure=True, log_msg=msg) def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) From e08d0eb8f97d6adadf49f9ece6bbeebfc30ebc51 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 22 Nov 2016 15:11:25 -0500 Subject: [PATCH 9/9] _log_and_render_error --- lbrynet/lbrynet_daemon/auth/server.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lbrynet/lbrynet_daemon/auth/server.py b/lbrynet/lbrynet_daemon/auth/server.py index 3ae8d7ff9..95c552ac8 100644 --- a/lbrynet/lbrynet_daemon/auth/server.py +++ b/lbrynet/lbrynet_daemon/auth/server.py @@ -95,17 +95,14 @@ class AuthJSONRPCServer(AuthorizedBase): def __init__(self, use_authentication=settings.use_auth_http): AuthorizedBase.__init__(self) self._use_authentication = use_authentication + self.announced_startup = False self.allowed_during_startup = [] self.sessions = {} def setup(self): return NotImplementedError() - def _render_error(self, failure, request, version=jsonrpclib.VERSION_1, response_code=FAILURE, - log_failure=False, log_msg=None): - if log_failure: - msg = log_msg or "API Failure: %s" - log_support.failure(Failure(failure), log, msg) + def _render_error(self, failure, request, version=jsonrpclib.VERSION_1, response_code=FAILURE): err = JSONRPCException(Failure(failure), response_code) fault = jsonrpclib.dumps(err, version=version) self._set_headers(request, fault) @@ -114,6 +111,11 @@ class AuthJSONRPCServer(AuthorizedBase): request.write(fault) request.finish() + def _log_and_render_error(self, failure, request, message=None, **kwargs): + msg = message or "API Failure: %s" + log_support.failure(Failure(failure), log, msg) + self._render_error(failure, request, **kwargs) + def render(self, request): notify_finish = request.notifyFinish() assert self._check_headers(request), InvalidHeaderError @@ -163,8 +165,7 @@ class AuthJSONRPCServer(AuthorizedBase): self._verify_token(session_id, parsed, token) except InvalidAuthenticationToken as err: log.warning("API validation failed") - self._render_error(err, request, version, - response_code=AuthJSONRPCServer.UNAUTHORIZED) + self._render_error(err, request, version=version, response_code=AuthJSONRPCServer.UNAUTHORIZED) return server.NOT_DONE_YET self._update_session_secret(session_id) reply_with_next_secret = True @@ -184,7 +185,7 @@ class AuthJSONRPCServer(AuthorizedBase): # cancel the response if the connection is broken notify_finish.addErrback(self._response_failed, d) d.addCallback(self._callback_render, request, version, reply_with_next_secret) - d.addErrback(self._render_error, request, version, log_failure=True) + d.addErrback(self._log_and_render_error, request, version=version) return server.NOT_DONE_YET def _register_user_session(self, session_id): @@ -285,8 +286,7 @@ class AuthJSONRPCServer(AuthorizedBase): self._render_message(request, encoded_message) except Exception as err: msg = "Failed to render API response: %s" - self._render_error(err, request, response_code=self.FAILURE, version=version, - log_failure=True, log_msg=msg) + self._log_and_render_error(err, request, message=msg, version=version) def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code})