From a89306b6bfc0ea0461eb427cfbcbe0c8f22fc617 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 20 Jul 2018 16:46:15 -0400 Subject: [PATCH] reorganize daemon startup -fix loggly not using the share usage setting -delete more --- lbrynet/core/log_support.py | 2 + lbrynet/daemon/Daemon.py | 33 ++------ lbrynet/daemon/DaemonConsole.py | 14 +--- lbrynet/daemon/DaemonControl.py | 29 ++----- lbrynet/daemon/DaemonServer.py | 77 ------------------- lbrynet/daemon/auth/factory.py | 38 +++++++++ lbrynet/daemon/auth/server.py | 53 ++++++++----- .../unit/lbrynet_daemon/auth/test_server.py | 2 +- 8 files changed, 87 insertions(+), 161 deletions(-) delete mode 100644 lbrynet/daemon/DaemonServer.py create mode 100644 lbrynet/daemon/auth/factory.py diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index a623c8b81..add93ea84 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -118,6 +118,8 @@ def get_loggly_url(token=None, version=None): def configure_loggly_handler(): if build_type.BUILD == 'dev': return + if not conf.settings['share_usage_data']: + return level = logging.ERROR handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id, session_id=conf.settings.get_session_id()) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index ce832de28..11a6d91fc 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -3,7 +3,6 @@ import binascii import logging.handlers import mimetypes import os -import base58 import requests import urllib import json @@ -27,7 +26,6 @@ from lbryschema.decode import smart_decode from lbrynet.core.system_info import get_lbrynet_version from lbrynet import conf from lbrynet.reflector import reupload -from lbrynet.core.log_support import configure_loggly_handler from lbrynet.daemon.Component import ComponentManager from lbrynet.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, SESSION_COMPONENT, DHT_COMPONENT from lbrynet.daemon.Components import STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT @@ -78,6 +76,7 @@ DIRECTION_ASCENDING = 'asc' DIRECTION_DESCENDING = 'desc' DIRECTIONS = DIRECTION_ASCENDING, DIRECTION_DESCENDING + class IterableContainer(object): def __iter__(self): for attr in dir(self): @@ -153,12 +152,10 @@ class Daemon(AuthJSONRPCServer): LBRYnet daemon, a jsonrpc interface to lbry functions """ - def __init__(self, analytics_manager, component_manager=None): - AuthJSONRPCServer.__init__(self, conf.settings['use_auth_http']) - self.analytics_manager = analytics_manager + def __init__(self, analytics_manager=None, component_manager=None): + AuthJSONRPCServer.__init__(self, analytics_manager, conf.settings['use_auth_http']) self.looping_call_manager = LoopingCallManager({ Checker.INTERNET_CONNECTION: LoopingCall(CheckInternetConnection(self)), - Checker.CONNECTION_STATUS: LoopingCall(self._update_connection_status), }) self.component_manager = component_manager or ComponentManager( analytics_manager=self.analytics_manager, @@ -185,11 +182,9 @@ class Daemon(AuthJSONRPCServer): @defer.inlineCallbacks def setup(self): reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) - configure_loggly_handler() if not self.analytics_manager.is_started: self.analytics_manager.start() self.looping_call_manager.start(Checker.INTERNET_CONNECTION, 3600) - self.looping_call_manager.start(Checker.CONNECTION_STATUS, 30) components = { EXCHANGE_RATE_MANAGER_COMPONENT: "exchange_rate_manager", @@ -205,18 +200,8 @@ class Daemon(AuthJSONRPCServer): log.info("Platform: %s", json.dumps(system_info.get_platform())) yield self.component_manager.setup(**{n: lambda _, c: setattr(self, components[c.component_name], c.component) for n in components.keys()}) - log.info("Started lbrynet-daemon") - def _check_network_connection(self): - self.connected_to_internet = utils.check_connection() - - def _update_connection_status(self): - self.connection_status_code = CONNECTION_STATUS_CONNECTED - - if not self.connected_to_internet: - self.connection_status_code = CONNECTION_STATUS_NETWORK - @staticmethod def _already_shutting_down(sig_num, frame): log.info("Already shutting down") @@ -603,7 +588,6 @@ class Daemon(AuthJSONRPCServer): direction = pieces[0] return field, direction - def _get_single_peer_downloader(self): downloader = SinglePeerDownloader() downloader.setup(self.wallet) @@ -706,19 +690,16 @@ class Daemon(AuthJSONRPCServer): wallet_is_encrypted = has_wallet and self.wallet.wallet and \ self.wallet.wallet.use_encryption + connection_code = CONNECTION_STATUS_CONNECTED if utils.check_connection() else CONNECTION_STATUS_NETWORK response = { 'lbry_id': base58.b58encode(self.node_id), 'installation_id': conf.settings.installation_id, - 'is_running': self.announced_startup, + 'is_running': all(self.component_manager.get_components_status().values()), 'is_first_run': self.wallet.is_first_run if has_wallet else None, 'startup_status': self.component_manager.get_components_status(), 'connection_status': { - 'code': self.connection_status_code, - 'message': ( - CONNECTION_MESSAGES[self.connection_status_code] - if self.connection_status_code is not None - else '' - ), + 'code': connection_code, + 'message': CONNECTION_MESSAGES[connection_code], }, 'wallet_is_encrypted': wallet_is_encrypted, 'blocks_behind': remote_height - local_height, # deprecated. remove from UI, then here diff --git a/lbrynet/daemon/DaemonConsole.py b/lbrynet/daemon/DaemonConsole.py index 6210dfc0e..65442e751 100644 --- a/lbrynet/daemon/DaemonConsole.py +++ b/lbrynet/daemon/DaemonConsole.py @@ -10,7 +10,6 @@ from lbrynet import analytics from lbrynet import conf from lbrynet.core import utils from lbrynet.core import log_support -from lbrynet.daemon.DaemonServer import DaemonServer from lbrynet.daemon.auth.client import LBRYAPIClient from lbrynet.daemon.Daemon import Daemon @@ -175,18 +174,7 @@ def start_server_and_listen(use_auth, analytics_manager, quiet): logging.getLogger("requests").setLevel(logging.CRITICAL) analytics_manager.send_server_startup() - daemon_server = DaemonServer(analytics_manager) - try: - yield daemon_server.start(use_auth) - analytics_manager.send_server_startup_success() - if not quiet: - print "Started lbrynet-daemon!" - defer.returnValue(True) - except Exception as e: - log.exception('Failed to start lbrynet-daemon') - analytics_manager.send_server_startup_error(str(e)) - daemon_server.stop() - raise + yield Daemon().start_listening() def threaded_terminal(started_daemon, quiet): diff --git a/lbrynet/daemon/DaemonControl.py b/lbrynet/daemon/DaemonControl.py index 8d73c9ce0..8db0511b9 100644 --- a/lbrynet/daemon/DaemonControl.py +++ b/lbrynet/daemon/DaemonControl.py @@ -12,13 +12,12 @@ from lbrynet.core import log_support import argparse import logging.handlers -from twisted.internet import defer, reactor +from twisted.internet import reactor from jsonrpc.proxy import JSONRPCProxy -from lbrynet import analytics from lbrynet import conf from lbrynet.core import utils, system_info -from lbrynet.daemon.DaemonServer import DaemonServer +from lbrynet.daemon.Daemon import Daemon log = logging.getLogger(__name__) @@ -71,6 +70,7 @@ def start(): lbrynet_log = conf.settings.get_log_filename() log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose) + log_support.configure_loggly_handler() log.debug('Final Settings: %s', conf.settings.get_current_settings_dict()) try: @@ -84,8 +84,8 @@ def start(): log.info("Starting lbrynet-daemon from command line") if test_internet_connection(): - analytics_manager = analytics.Manager.new_instance() - start_server_and_listen(analytics_manager) + daemon = Daemon() + daemon.start_listening() reactor.run() else: log.info("Not connected to internet, unable to start") @@ -101,24 +101,5 @@ def update_settings_from_args(args): }, data_types=(conf.TYPE_CLI,)) - -@defer.inlineCallbacks -def start_server_and_listen(analytics_manager): - """ - Args: - use_auth: set to true to enable http authentication - analytics_manager: to send analytics - """ - analytics_manager.send_server_startup() - daemon_server = DaemonServer(analytics_manager) - try: - yield daemon_server.start(conf.settings['use_auth_http']) - analytics_manager.send_server_startup_success() - except Exception as e: - log.exception('Failed to start lbrynet-daemon') - analytics_manager.send_server_startup_error(str(e)) - daemon_server.stop() - - if __name__ == "__main__": start() diff --git a/lbrynet/daemon/DaemonServer.py b/lbrynet/daemon/DaemonServer.py deleted file mode 100644 index e8c00606b..000000000 --- a/lbrynet/daemon/DaemonServer.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -import os - -from twisted.web import server, guard, resource -from twisted.internet import defer, reactor, error -from twisted.cred import portal - -from lbrynet import conf -from lbrynet.daemon.Daemon import Daemon -from lbrynet.daemon.auth.auth import PasswordChecker, HttpPasswordRealm -from lbrynet.daemon.auth.util import initialize_api_key_file - -log = logging.getLogger(__name__) - - -class IndexResource(resource.Resource): - def getChild(self, name, request): - request.setHeader('cache-control', 'no-cache, no-store, must-revalidate') - request.setHeader('expires', '0') - return self if name == '' else resource.Resource.getChild(self, name, request) - - -class DaemonServer(object): - def __init__(self, analytics_manager=None): - self._daemon = None - self.root = None - self.server_port = None - self.analytics_manager = analytics_manager - - def _setup_server(self, use_auth): - self.root = IndexResource() - self._daemon = Daemon(self.analytics_manager) - self.root.putChild("", self._daemon) - # TODO: DEPRECATED, remove this and just serve the API at the root - self.root.putChild(conf.settings['API_ADDRESS'], self._daemon) - - lbrynet_server = get_site_base(use_auth, self.root) - - try: - self.server_port = reactor.listenTCP( - conf.settings['api_port'], lbrynet_server, interface=conf.settings['api_host']) - log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port']) - except error.CannotListenError: - log.info('Daemon already running, exiting app') - raise - - return defer.succeed(True) - - @defer.inlineCallbacks - def start(self, use_auth): - yield self._setup_server(use_auth) - yield self._daemon.setup() - - def stop(self): - if reactor.running: - log.info("Stopping the reactor") - reactor.fireSystemEvent("shutdown") - - -def get_site_base(use_auth, root): - if use_auth: - log.info("Using authenticated API") - root = create_auth_session(root) - else: - log.info("Using non-authenticated API") - return server.Site(root) - - -def create_auth_session(root): - pw_path = os.path.join(conf.settings['data_dir'], ".api_keys") - initialize_api_key_file(pw_path) - checker = PasswordChecker.load_file(pw_path) - realm = HttpPasswordRealm(root) - portal_to_realm = portal.Portal(realm, [checker, ]) - factory = guard.BasicCredentialFactory('Login to lbrynet api') - _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) - return _lbrynet_server diff --git a/lbrynet/daemon/auth/factory.py b/lbrynet/daemon/auth/factory.py new file mode 100644 index 000000000..fed157cc0 --- /dev/null +++ b/lbrynet/daemon/auth/factory.py @@ -0,0 +1,38 @@ +import logging +import os + +from twisted.web import server, guard, resource +from twisted.cred import portal + +from lbrynet import conf +from .auth import PasswordChecker, HttpPasswordRealm +from .util import initialize_api_key_file + +log = logging.getLogger(__name__) + + +class AuthJSONRPCResource(resource.Resource): + def __init__(self, protocol): + resource.Resource.__init__(self) + self.putChild("", protocol) + self.putChild(conf.settings['API_ADDRESS'], protocol) + + def getChild(self, name, request): + request.setHeader('cache-control', 'no-cache, no-store, must-revalidate') + request.setHeader('expires', '0') + return self if name == '' else resource.Resource.getChild(self, name, request) + + def getServerFactory(self): + if conf.settings['use_auth_http']: + log.info("Using authenticated API") + pw_path = os.path.join(conf.settings['data_dir'], ".api_keys") + initialize_api_key_file(pw_path) + checker = PasswordChecker.load_file(pw_path) + realm = HttpPasswordRealm(self) + portal_to_realm = portal.Portal(realm, [checker, ]) + factory = guard.BasicCredentialFactory('Login to lbrynet api') + root = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) + else: + log.info("Using non-authenticated API") + root = self + return server.Site(root) diff --git a/lbrynet/daemon/auth/server.py b/lbrynet/daemon/auth/server.py index af2461839..f71a1826c 100644 --- a/lbrynet/daemon/auth/server.py +++ b/lbrynet/daemon/auth/server.py @@ -13,14 +13,14 @@ from twisted.internet.error import ConnectionDone, ConnectionLost from txjsonrpc import jsonrpclib from traceback import format_exc -from lbrynet import conf +from lbrynet import conf, analytics from lbrynet.core.Error import InvalidAuthenticationToken from lbrynet.core import utils from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet -from lbrynet.daemon.auth.util import APIKey, get_auth_message -from lbrynet.daemon.auth.client import LBRY_SECRET from lbrynet.undecorated import undecorated - +from .util import APIKey, get_auth_message +from .client import LBRY_SECRET +from .factory import AuthJSONRPCResource log = logging.getLogger(__name__) EMPTY_PARAMS = [{}] @@ -93,10 +93,6 @@ class UnknownAPIMethodError(Exception): pass -class NotAllowedDuringStartupError(Exception): - pass - - def trap(err, *to_trap): err.trap(*to_trap) @@ -197,13 +193,37 @@ class AuthJSONRPCServer(AuthorizedBase): isLeaf = True allowed_during_startup = [] - def __init__(self, use_authentication=None): + def __init__(self, analytics_manager, use_authentication=None): + self.analytics_manager = analytics_manager or analytics.Manager.new_instance() self._use_authentication = use_authentication or conf.settings['use_auth_http'] self.announced_startup = False self.sessions = {} + @defer.inlineCallbacks + def start_listening(self): + from twisted.internet import reactor, error as tx_error + + try: + reactor.listenTCP( + conf.settings['api_port'], self.get_server_factory(), interface=conf.settings['api_host'] + ) + log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port']) + yield self.setup() + self.analytics_manager.send_server_startup_success() + except tx_error.CannotListenError: + log.error('lbrynet API failed to bind TCP %s:%i for listening', conf.settings['api_host'], + conf.settings['api_port']) + reactor.fireSystemEvent("shutdown") + except Exception as err: + self.analytics_manager.send_server_startup_error(str(err)) + log.exception('Failed to start lbrynet-daemon') + reactor.fireSystemEvent("shutdown") + def setup(self): - return NotImplementedError() + raise NotImplementedError() + + def get_server_factory(self): + return AuthJSONRPCResource(self).getServerFactory() def _set_headers(self, request, data, update_secret=False): if conf.settings['allowed_origin']: @@ -233,8 +253,9 @@ class AuthJSONRPCServer(AuthorizedBase): else: # last resort, just cast it as a string error = JSONRPCError(str(failure)) - log.warning("error processing api request: %s\ntraceback: %s", error.message, - "\n".join(error.traceback)) + if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet): + log.warning("error processing api request: %s\ntraceback: %s", error.message, + "\n".join(error.traceback)) response_content = jsonrpc_dumps_pretty(error, id=id_) self._set_headers(request, response_content) request.setResponseCode(200) @@ -330,14 +351,6 @@ class AuthJSONRPCServer(AuthorizedBase): request, request_id ) return server.NOT_DONE_YET - except NotAllowedDuringStartupError: - log.warning('Function not allowed during startup: %s', function_name) - self._render_error( - JSONRPCError("This method is unavailable until the daemon is fully started", - code=JSONRPCError.CODE_INVALID_REQUEST), - request, request_id - ) - return server.NOT_DONE_YET if args == EMPTY_PARAMS or args == []: _args, _kwargs = (), {} diff --git a/lbrynet/tests/unit/lbrynet_daemon/auth/test_server.py b/lbrynet/tests/unit/lbrynet_daemon/auth/test_server.py index 80fa4aa7c..bd1d5399e 100644 --- a/lbrynet/tests/unit/lbrynet_daemon/auth/test_server.py +++ b/lbrynet/tests/unit/lbrynet_daemon/auth/test_server.py @@ -11,7 +11,7 @@ class AuthJSONRPCServerTest(unittest.TestCase): # onto it. def setUp(self): conf.initialize_settings(False) - self.server = server.AuthJSONRPCServer(use_authentication=False) + self.server = server.AuthJSONRPCServer(True, use_authentication=False) def test_get_server_port(self): self.assertSequenceEqual(