reorganize daemon startup

-fix loggly not using the share usage setting

-delete more

lbrynet/core/log_support.py

@@ -118,6 +118,8 @@ def get_loggly_url(token=None, version=None):
 def configure_loggly_handler():
     if build_type.BUILD == 'dev':
         return
+    if not conf.settings['share_usage_data']:
+        return
     level = logging.ERROR
     handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
                                  session_id=conf.settings.get_session_id())

lbrynet/daemon/Daemon.py

@@ -3,7 +3,6 @@ import binascii
 import logging.handlers
 import mimetypes
 import os
-import base58
 import requests
 import urllib
 import json
@@ -27,7 +26,6 @@ from lbryschema.decode import smart_decode
 from lbrynet.core.system_info import get_lbrynet_version
 from lbrynet import conf
 from lbrynet.reflector import reupload
-from lbrynet.core.log_support import configure_loggly_handler
 from lbrynet.daemon.Component import ComponentManager
 from lbrynet.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, SESSION_COMPONENT, DHT_COMPONENT
 from lbrynet.daemon.Components import STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT
@@ -78,6 +76,7 @@ DIRECTION_ASCENDING = 'asc'
 DIRECTION_DESCENDING = 'desc'
 DIRECTIONS = DIRECTION_ASCENDING, DIRECTION_DESCENDING
 
+
 class IterableContainer(object):
     def __iter__(self):
         for attr in dir(self):
@@ -153,12 +152,10 @@ class Daemon(AuthJSONRPCServer):
     LBRYnet daemon, a jsonrpc interface to lbry functions
     """
 
-    def __init__(self, analytics_manager, component_manager=None):
+    def __init__(self, analytics_manager=None, component_manager=None):
-        AuthJSONRPCServer.__init__(self, conf.settings['use_auth_http'])
+        AuthJSONRPCServer.__init__(self, analytics_manager, conf.settings['use_auth_http'])
-        self.analytics_manager = analytics_manager
         self.looping_call_manager = LoopingCallManager({
             Checker.INTERNET_CONNECTION: LoopingCall(CheckInternetConnection(self)),
-            Checker.CONNECTION_STATUS: LoopingCall(self._update_connection_status),
         })
         self.component_manager = component_manager or ComponentManager(
             analytics_manager=self.analytics_manager,
@@ -185,11 +182,9 @@ class Daemon(AuthJSONRPCServer):
     @defer.inlineCallbacks
     def setup(self):
         reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
-        configure_loggly_handler()
         if not self.analytics_manager.is_started:
             self.analytics_manager.start()
         self.looping_call_manager.start(Checker.INTERNET_CONNECTION, 3600)
-        self.looping_call_manager.start(Checker.CONNECTION_STATUS, 30)
 
         components = {
             EXCHANGE_RATE_MANAGER_COMPONENT: "exchange_rate_manager",
@@ -205,18 +200,8 @@ class Daemon(AuthJSONRPCServer):
         log.info("Platform: %s", json.dumps(system_info.get_platform()))
         yield self.component_manager.setup(**{n: lambda _, c: setattr(self, components[c.component_name], c.component)
                                               for n in components.keys()})
-
         log.info("Started lbrynet-daemon")
 
-    def _check_network_connection(self):
-        self.connected_to_internet = utils.check_connection()
-
-    def _update_connection_status(self):
-        self.connection_status_code = CONNECTION_STATUS_CONNECTED
-
-        if not self.connected_to_internet:
-            self.connection_status_code = CONNECTION_STATUS_NETWORK
-
     @staticmethod
     def _already_shutting_down(sig_num, frame):
         log.info("Already shutting down")
@@ -603,7 +588,6 @@ class Daemon(AuthJSONRPCServer):
             direction = pieces[0]
         return field, direction
 
-
     def _get_single_peer_downloader(self):
         downloader = SinglePeerDownloader()
         downloader.setup(self.wallet)
@@ -706,19 +690,16 @@ class Daemon(AuthJSONRPCServer):
         wallet_is_encrypted = has_wallet and self.wallet.wallet and \
                               self.wallet.wallet.use_encryption
 
+        connection_code = CONNECTION_STATUS_CONNECTED if utils.check_connection() else CONNECTION_STATUS_NETWORK
         response = {
             'lbry_id': base58.b58encode(self.node_id),
             'installation_id': conf.settings.installation_id,
-            'is_running': self.announced_startup,
+            'is_running': all(self.component_manager.get_components_status().values()),
             'is_first_run': self.wallet.is_first_run if has_wallet else None,
             'startup_status': self.component_manager.get_components_status(),
             'connection_status': {
-                'code': self.connection_status_code,
+                'code': connection_code,
-                'message': (
+                'message': CONNECTION_MESSAGES[connection_code],
-                    CONNECTION_MESSAGES[self.connection_status_code]
-                    if self.connection_status_code is not None
-                    else ''
-                ),
             },
             'wallet_is_encrypted': wallet_is_encrypted,
             'blocks_behind': remote_height - local_height,  # deprecated. remove from UI, then here

lbrynet/daemon/DaemonConsole.py

@@ -10,7 +10,6 @@ from lbrynet import analytics
 from lbrynet import conf
 from lbrynet.core import utils
 from lbrynet.core import log_support
-from lbrynet.daemon.DaemonServer import DaemonServer
 from lbrynet.daemon.auth.client import LBRYAPIClient
 from lbrynet.daemon.Daemon import Daemon
 
@@ -175,18 +174,7 @@ def start_server_and_listen(use_auth, analytics_manager, quiet):
     logging.getLogger("requests").setLevel(logging.CRITICAL)
 
     analytics_manager.send_server_startup()
-    daemon_server = DaemonServer(analytics_manager)
+    yield Daemon().start_listening()
-    try:
-        yield daemon_server.start(use_auth)
-        analytics_manager.send_server_startup_success()
-        if not quiet:
-            print "Started lbrynet-daemon!"
-        defer.returnValue(True)
-    except Exception as e:
-        log.exception('Failed to start lbrynet-daemon')
-        analytics_manager.send_server_startup_error(str(e))
-        daemon_server.stop()
-        raise
 
 
 def threaded_terminal(started_daemon, quiet):

lbrynet/daemon/DaemonControl.py

@@ -12,13 +12,12 @@ from lbrynet.core import log_support
 import argparse
 import logging.handlers
 
-from twisted.internet import defer, reactor
+from twisted.internet import reactor
 from jsonrpc.proxy import JSONRPCProxy
 
-from lbrynet import analytics
 from lbrynet import conf
 from lbrynet.core import utils, system_info
-from lbrynet.daemon.DaemonServer import DaemonServer
+from lbrynet.daemon.Daemon import Daemon
 
 log = logging.getLogger(__name__)
 
@@ -71,6 +70,7 @@ def start():
 
     lbrynet_log = conf.settings.get_log_filename()
     log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
+    log_support.configure_loggly_handler()
     log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
 
     try:
@@ -84,8 +84,8 @@ def start():
     log.info("Starting lbrynet-daemon from command line")
 
     if test_internet_connection():
-        analytics_manager = analytics.Manager.new_instance()
+        daemon = Daemon()
-        start_server_and_listen(analytics_manager)
+        daemon.start_listening()
         reactor.run()
     else:
         log.info("Not connected to internet, unable to start")
@@ -101,24 +101,5 @@ def update_settings_from_args(args):
         }, data_types=(conf.TYPE_CLI,))
 
 
-
-@defer.inlineCallbacks
-def start_server_and_listen(analytics_manager):
-    """
-    Args:
-        use_auth: set to true to enable http authentication
-        analytics_manager: to send analytics
-    """
-    analytics_manager.send_server_startup()
-    daemon_server = DaemonServer(analytics_manager)
-    try:
-        yield daemon_server.start(conf.settings['use_auth_http'])
-        analytics_manager.send_server_startup_success()
-    except Exception as e:
-        log.exception('Failed to start lbrynet-daemon')
-        analytics_manager.send_server_startup_error(str(e))
-        daemon_server.stop()
-
-
 if __name__ == "__main__":
     start()

lbrynet/daemon/DaemonServer.py

@@ -1,77 +0,0 @@
-import logging
-import os
-
-from twisted.web import server, guard, resource
-from twisted.internet import defer, reactor, error
-from twisted.cred import portal
-
-from lbrynet import conf
-from lbrynet.daemon.Daemon import Daemon
-from lbrynet.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
-from lbrynet.daemon.auth.util import initialize_api_key_file
-
-log = logging.getLogger(__name__)
-
-
-class IndexResource(resource.Resource):
-    def getChild(self, name, request):
-        request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
-        request.setHeader('expires', '0')
-        return self if name == '' else resource.Resource.getChild(self, name, request)
-
-
-class DaemonServer(object):
-    def __init__(self, analytics_manager=None):
-        self._daemon = None
-        self.root = None
-        self.server_port = None
-        self.analytics_manager = analytics_manager
-
-    def _setup_server(self, use_auth):
-        self.root = IndexResource()
-        self._daemon = Daemon(self.analytics_manager)
-        self.root.putChild("", self._daemon)
-        # TODO: DEPRECATED, remove this and just serve the API at the root
-        self.root.putChild(conf.settings['API_ADDRESS'], self._daemon)
-
-        lbrynet_server = get_site_base(use_auth, self.root)
-
-        try:
-            self.server_port = reactor.listenTCP(
-                conf.settings['api_port'], lbrynet_server, interface=conf.settings['api_host'])
-            log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
-        except error.CannotListenError:
-            log.info('Daemon already running, exiting app')
-            raise
-
-        return defer.succeed(True)
-
-    @defer.inlineCallbacks
-    def start(self, use_auth):
-        yield self._setup_server(use_auth)
-        yield self._daemon.setup()
-
-    def stop(self):
-        if reactor.running:
-            log.info("Stopping the reactor")
-            reactor.fireSystemEvent("shutdown")
-
-
-def get_site_base(use_auth, root):
-    if use_auth:
-        log.info("Using authenticated API")
-        root = create_auth_session(root)
-    else:
-        log.info("Using non-authenticated API")
-    return server.Site(root)
-
-
-def create_auth_session(root):
-    pw_path = os.path.join(conf.settings['data_dir'], ".api_keys")
-    initialize_api_key_file(pw_path)
-    checker = PasswordChecker.load_file(pw_path)
-    realm = HttpPasswordRealm(root)
-    portal_to_realm = portal.Portal(realm, [checker, ])
-    factory = guard.BasicCredentialFactory('Login to lbrynet api')
-    _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
-    return _lbrynet_server

lbrynet/daemon/auth/factory.py

@@ -0,0 +1,38 @@
+import logging
+import os
+
+from twisted.web import server, guard, resource
+from twisted.cred import portal
+
+from lbrynet import conf
+from .auth import PasswordChecker, HttpPasswordRealm
+from .util import initialize_api_key_file
+
+log = logging.getLogger(__name__)
+
+
+class AuthJSONRPCResource(resource.Resource):
+    def __init__(self, protocol):
+        resource.Resource.__init__(self)
+        self.putChild("", protocol)
+        self.putChild(conf.settings['API_ADDRESS'], protocol)
+
+    def getChild(self, name, request):
+        request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
+        request.setHeader('expires', '0')
+        return self if name == '' else resource.Resource.getChild(self, name, request)
+
+    def getServerFactory(self):
+        if conf.settings['use_auth_http']:
+            log.info("Using authenticated API")
+            pw_path = os.path.join(conf.settings['data_dir'], ".api_keys")
+            initialize_api_key_file(pw_path)
+            checker = PasswordChecker.load_file(pw_path)
+            realm = HttpPasswordRealm(self)
+            portal_to_realm = portal.Portal(realm, [checker, ])
+            factory = guard.BasicCredentialFactory('Login to lbrynet api')
+            root = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
+        else:
+            log.info("Using non-authenticated API")
+            root = self
+        return server.Site(root)

lbrynet/daemon/auth/server.py

@@ -13,14 +13,14 @@ from twisted.internet.error import ConnectionDone, ConnectionLost
 from txjsonrpc import jsonrpclib
 from traceback import format_exc
 
-from lbrynet import conf
+from lbrynet import conf, analytics
 from lbrynet.core.Error import InvalidAuthenticationToken
 from lbrynet.core import utils
 from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet
-from lbrynet.daemon.auth.util import APIKey, get_auth_message
-from lbrynet.daemon.auth.client import LBRY_SECRET
 from lbrynet.undecorated import undecorated
-
+from .util import APIKey, get_auth_message
+from .client import LBRY_SECRET
+from .factory import AuthJSONRPCResource
 log = logging.getLogger(__name__)
 
 EMPTY_PARAMS = [{}]
@@ -93,10 +93,6 @@ class UnknownAPIMethodError(Exception):
     pass
 
 
-class NotAllowedDuringStartupError(Exception):
-    pass
-
-
 def trap(err, *to_trap):
     err.trap(*to_trap)
 
@@ -197,13 +193,37 @@ class AuthJSONRPCServer(AuthorizedBase):
     isLeaf = True
     allowed_during_startup = []
 
-    def __init__(self, use_authentication=None):
+    def __init__(self, analytics_manager, use_authentication=None):
+        self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
         self._use_authentication = use_authentication or conf.settings['use_auth_http']
         self.announced_startup = False
         self.sessions = {}
 
+    @defer.inlineCallbacks
+    def start_listening(self):
+        from twisted.internet import reactor, error as tx_error
+
+        try:
+            reactor.listenTCP(
+                conf.settings['api_port'], self.get_server_factory(), interface=conf.settings['api_host']
+            )
+            log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
+            yield self.setup()
+            self.analytics_manager.send_server_startup_success()
+        except tx_error.CannotListenError:
+            log.error('lbrynet API failed to bind TCP %s:%i for listening', conf.settings['api_host'],
+                      conf.settings['api_port'])
+            reactor.fireSystemEvent("shutdown")
+        except Exception as err:
+            self.analytics_manager.send_server_startup_error(str(err))
+            log.exception('Failed to start lbrynet-daemon')
+            reactor.fireSystemEvent("shutdown")
+
     def setup(self):
-        return NotImplementedError()
+        raise NotImplementedError()
+
+    def get_server_factory(self):
+        return AuthJSONRPCResource(self).getServerFactory()
 
     def _set_headers(self, request, data, update_secret=False):
         if conf.settings['allowed_origin']:
@@ -233,8 +253,9 @@ class AuthJSONRPCServer(AuthorizedBase):
         else:
             # last resort, just cast it as a string
             error = JSONRPCError(str(failure))
-        log.warning("error processing api request: %s\ntraceback: %s", error.message,
+        if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
-                    "\n".join(error.traceback))
+            log.warning("error processing api request: %s\ntraceback: %s", error.message,
+                        "\n".join(error.traceback))
         response_content = jsonrpc_dumps_pretty(error, id=id_)
         self._set_headers(request, response_content)
         request.setResponseCode(200)
@@ -330,14 +351,6 @@ class AuthJSONRPCServer(AuthorizedBase):
                 request, request_id
             )
             return server.NOT_DONE_YET
-        except NotAllowedDuringStartupError:
-            log.warning('Function not allowed during startup: %s', function_name)
-            self._render_error(
-                JSONRPCError("This method is unavailable until the daemon is fully started",
-                             code=JSONRPCError.CODE_INVALID_REQUEST),
-                request, request_id
-            )
-            return server.NOT_DONE_YET
 
         if args == EMPTY_PARAMS or args == []:
             _args, _kwargs = (), {}

lbrynet/tests/unit/lbrynet_daemon/auth/test_server.py

@@ -11,7 +11,7 @@ class AuthJSONRPCServerTest(unittest.TestCase):
     # onto it.
     def setUp(self):
         conf.initialize_settings(False)
-        self.server = server.AuthJSONRPCServer(use_authentication=False)
+        self.server = server.AuthJSONRPCServer(True, use_authentication=False)
 
     def test_get_server_port(self):
         self.assertSequenceEqual(