forked from LBRYCommunity/lbry-sdk
reorganize daemon startup
-fix loggly not using the share usage setting -delete more
This commit is contained in:
Jack Robison
parent
c3120e93cf
commit
a89306b6bf
8 changed files with 87 additions and 161 deletions
|
|
@ -118,6 +118,8 @@ def get_loggly_url(token=None, version=None):
|
|||
def configure_loggly_handler():
|
||||
if build_type.BUILD == 'dev':
|
||||
return
|
||||
if not conf.settings['share_usage_data']:
|
||||
return
|
||||
level = logging.ERROR
|
||||
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
|
||||
session_id=conf.settings.get_session_id())
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import binascii
|
|||
import logging.handlers
|
||||
import mimetypes
|
||||
import os
|
||||
import base58
|
||||
import requests
|
||||
import urllib
|
||||
import json
|
||||
|
|
@ -27,7 +26,6 @@ from lbryschema.decode import smart_decode
|
|||
from lbrynet.core.system_info import get_lbrynet_version
|
||||
from lbrynet import conf
|
||||
from lbrynet.reflector import reupload
|
||||
from lbrynet.core.log_support import configure_loggly_handler
|
||||
from lbrynet.daemon.Component import ComponentManager
|
||||
from lbrynet.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, SESSION_COMPONENT, DHT_COMPONENT
|
||||
from lbrynet.daemon.Components import STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT
|
||||
|
|
@ -78,6 +76,7 @@ DIRECTION_ASCENDING = 'asc'
|
|||
DIRECTION_DESCENDING = 'desc'
|
||||
DIRECTIONS = DIRECTION_ASCENDING, DIRECTION_DESCENDING
|
||||
|
||||
|
||||
class IterableContainer(object):
|
||||
def __iter__(self):
|
||||
for attr in dir(self):
|
||||
|
|
@ -153,12 +152,10 @@ class Daemon(AuthJSONRPCServer):
|
|||
LBRYnet daemon, a jsonrpc interface to lbry functions
|
||||
"""
|
||||
|
||||
def __init__(self, analytics_manager, component_manager=None):
|
||||
AuthJSONRPCServer.__init__(self, conf.settings['use_auth_http'])
|
||||
self.analytics_manager = analytics_manager
|
||||
def __init__(self, analytics_manager=None, component_manager=None):
|
||||
AuthJSONRPCServer.__init__(self, analytics_manager, conf.settings['use_auth_http'])
|
||||
self.looping_call_manager = LoopingCallManager({
|
||||
Checker.INTERNET_CONNECTION: LoopingCall(CheckInternetConnection(self)),
|
||||
Checker.CONNECTION_STATUS: LoopingCall(self._update_connection_status),
|
||||
})
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
analytics_manager=self.analytics_manager,
|
||||
|
|
@ -185,11 +182,9 @@ class Daemon(AuthJSONRPCServer):
|
|||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
|
||||
configure_loggly_handler()
|
||||
if not self.analytics_manager.is_started:
|
||||
self.analytics_manager.start()
|
||||
self.looping_call_manager.start(Checker.INTERNET_CONNECTION, 3600)
|
||||
self.looping_call_manager.start(Checker.CONNECTION_STATUS, 30)
|
||||
|
||||
components = {
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT: "exchange_rate_manager",
|
||||
|
|
@ -205,18 +200,8 @@ class Daemon(AuthJSONRPCServer):
|
|||
log.info("Platform: %s", json.dumps(system_info.get_platform()))
|
||||
yield self.component_manager.setup(**{n: lambda _, c: setattr(self, components[c.component_name], c.component)
|
||||
for n in components.keys()})
|
||||
|
||||
log.info("Started lbrynet-daemon")
|
||||
|
||||
def _check_network_connection(self):
|
||||
self.connected_to_internet = utils.check_connection()
|
||||
|
||||
def _update_connection_status(self):
|
||||
self.connection_status_code = CONNECTION_STATUS_CONNECTED
|
||||
|
||||
if not self.connected_to_internet:
|
||||
self.connection_status_code = CONNECTION_STATUS_NETWORK
|
||||
|
||||
@staticmethod
|
||||
def _already_shutting_down(sig_num, frame):
|
||||
log.info("Already shutting down")
|
||||
|
|
@ -603,7 +588,6 @@ class Daemon(AuthJSONRPCServer):
|
|||
direction = pieces[0]
|
||||
return field, direction
|
||||
|
||||
|
||||
def _get_single_peer_downloader(self):
|
||||
downloader = SinglePeerDownloader()
|
||||
downloader.setup(self.wallet)
|
||||
|
|
@ -706,19 +690,16 @@ class Daemon(AuthJSONRPCServer):
|
|||
wallet_is_encrypted = has_wallet and self.wallet.wallet and \
|
||||
self.wallet.wallet.use_encryption
|
||||
|
||||
connection_code = CONNECTION_STATUS_CONNECTED if utils.check_connection() else CONNECTION_STATUS_NETWORK
|
||||
response = {
|
||||
'lbry_id': base58.b58encode(self.node_id),
|
||||
'installation_id': conf.settings.installation_id,
|
||||
'is_running': self.announced_startup,
|
||||
'is_running': all(self.component_manager.get_components_status().values()),
|
||||
'is_first_run': self.wallet.is_first_run if has_wallet else None,
|
||||
'startup_status': self.component_manager.get_components_status(),
|
||||
'connection_status': {
|
||||
'code': self.connection_status_code,
|
||||
'message': (
|
||||
CONNECTION_MESSAGES[self.connection_status_code]
|
||||
if self.connection_status_code is not None
|
||||
else ''
|
||||
),
|
||||
'code': connection_code,
|
||||
'message': CONNECTION_MESSAGES[connection_code],
|
||||
},
|
||||
'wallet_is_encrypted': wallet_is_encrypted,
|
||||
'blocks_behind': remote_height - local_height, # deprecated. remove from UI, then here
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from lbrynet import analytics
|
|||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core import log_support
|
||||
from lbrynet.daemon.DaemonServer import DaemonServer
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
|
||||
|
|
@ -175,18 +174,7 @@ def start_server_and_listen(use_auth, analytics_manager, quiet):
|
|||
logging.getLogger("requests").setLevel(logging.CRITICAL)
|
||||
|
||||
analytics_manager.send_server_startup()
|
||||
daemon_server = DaemonServer(analytics_manager)
|
||||
try:
|
||||
yield daemon_server.start(use_auth)
|
||||
analytics_manager.send_server_startup_success()
|
||||
if not quiet:
|
||||
print "Started lbrynet-daemon!"
|
||||
defer.returnValue(True)
|
||||
except Exception as e:
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
analytics_manager.send_server_startup_error(str(e))
|
||||
daemon_server.stop()
|
||||
raise
|
||||
yield Daemon().start_listening()
|
||||
|
||||
|
||||
def threaded_terminal(started_daemon, quiet):
|
||||
|
|
|
|||
|
|
@ -12,13 +12,12 @@ from lbrynet.core import log_support
|
|||
import argparse
|
||||
import logging.handlers
|
||||
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet import reactor
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet import analytics
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils, system_info
|
||||
from lbrynet.daemon.DaemonServer import DaemonServer
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -71,6 +70,7 @@ def start():
|
|||
|
||||
lbrynet_log = conf.settings.get_log_filename()
|
||||
log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
|
||||
log_support.configure_loggly_handler()
|
||||
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
|
||||
|
||||
try:
|
||||
|
|
@ -84,8 +84,8 @@ def start():
|
|||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if test_internet_connection():
|
||||
analytics_manager = analytics.Manager.new_instance()
|
||||
start_server_and_listen(analytics_manager)
|
||||
daemon = Daemon()
|
||||
daemon.start_listening()
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
|
@ -101,24 +101,5 @@ def update_settings_from_args(args):
|
|||
}, data_types=(conf.TYPE_CLI,))
|
||||
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_server_and_listen(analytics_manager):
|
||||
"""
|
||||
Args:
|
||||
use_auth: set to true to enable http authentication
|
||||
analytics_manager: to send analytics
|
||||
"""
|
||||
analytics_manager.send_server_startup()
|
||||
daemon_server = DaemonServer(analytics_manager)
|
||||
try:
|
||||
yield daemon_server.start(conf.settings['use_auth_http'])
|
||||
analytics_manager.send_server_startup_success()
|
||||
except Exception as e:
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
analytics_manager.send_server_startup_error(str(e))
|
||||
daemon_server.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start()
|
||||
|
|
|
|||
|
|
@ -1,77 +0,0 @@
|
|||
import logging
|
||||
import os
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.internet import defer, reactor, error
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
from lbrynet.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
|
||||
from lbrynet.daemon.auth.util import initialize_api_key_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IndexResource(resource.Resource):
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
|
||||
class DaemonServer(object):
|
||||
def __init__(self, analytics_manager=None):
|
||||
self._daemon = None
|
||||
self.root = None
|
||||
self.server_port = None
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
def _setup_server(self, use_auth):
|
||||
self.root = IndexResource()
|
||||
self._daemon = Daemon(self.analytics_manager)
|
||||
self.root.putChild("", self._daemon)
|
||||
# TODO: DEPRECATED, remove this and just serve the API at the root
|
||||
self.root.putChild(conf.settings['API_ADDRESS'], self._daemon)
|
||||
|
||||
lbrynet_server = get_site_base(use_auth, self.root)
|
||||
|
||||
try:
|
||||
self.server_port = reactor.listenTCP(
|
||||
conf.settings['api_port'], lbrynet_server, interface=conf.settings['api_host'])
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
except error.CannotListenError:
|
||||
log.info('Daemon already running, exiting app')
|
||||
raise
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, use_auth):
|
||||
yield self._setup_server(use_auth)
|
||||
yield self._daemon.setup()
|
||||
|
||||
def stop(self):
|
||||
if reactor.running:
|
||||
log.info("Stopping the reactor")
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
|
||||
def get_site_base(use_auth, root):
|
||||
if use_auth:
|
||||
log.info("Using authenticated API")
|
||||
root = create_auth_session(root)
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
return server.Site(root)
|
||||
|
||||
|
||||
def create_auth_session(root):
|
||||
pw_path = os.path.join(conf.settings['data_dir'], ".api_keys")
|
||||
initialize_api_key_file(pw_path)
|
||||
checker = PasswordChecker.load_file(pw_path)
|
||||
realm = HttpPasswordRealm(root)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
factory = guard.BasicCredentialFactory('Login to lbrynet api')
|
||||
_lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
|
||||
return _lbrynet_server
|
||||
38
lbrynet/daemon/auth/factory.py
Normal file
38
lbrynet/daemon/auth/factory.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import logging
|
||||
import os
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from .auth import PasswordChecker, HttpPasswordRealm
|
||||
from .util import initialize_api_key_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AuthJSONRPCResource(resource.Resource):
|
||||
def __init__(self, protocol):
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild("", protocol)
|
||||
self.putChild(conf.settings['API_ADDRESS'], protocol)
|
||||
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
def getServerFactory(self):
|
||||
if conf.settings['use_auth_http']:
|
||||
log.info("Using authenticated API")
|
||||
pw_path = os.path.join(conf.settings['data_dir'], ".api_keys")
|
||||
initialize_api_key_file(pw_path)
|
||||
checker = PasswordChecker.load_file(pw_path)
|
||||
realm = HttpPasswordRealm(self)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
factory = guard.BasicCredentialFactory('Login to lbrynet api')
|
||||
root = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
root = self
|
||||
return server.Site(root)
|
||||
|
|
@ -13,14 +13,14 @@ from twisted.internet.error import ConnectionDone, ConnectionLost
|
|||
from txjsonrpc import jsonrpclib
|
||||
from traceback import format_exc
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet import conf, analytics
|
||||
from lbrynet.core.Error import InvalidAuthenticationToken
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet
|
||||
from lbrynet.daemon.auth.util import APIKey, get_auth_message
|
||||
from lbrynet.daemon.auth.client import LBRY_SECRET
|
||||
from lbrynet.undecorated import undecorated
|
||||
|
||||
from .util import APIKey, get_auth_message
|
||||
from .client import LBRY_SECRET
|
||||
from .factory import AuthJSONRPCResource
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
EMPTY_PARAMS = [{}]
|
||||
|
|
@ -93,10 +93,6 @@ class UnknownAPIMethodError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class NotAllowedDuringStartupError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def trap(err, *to_trap):
|
||||
err.trap(*to_trap)
|
||||
|
||||
|
|
@ -197,13 +193,37 @@ class AuthJSONRPCServer(AuthorizedBase):
|
|||
isLeaf = True
|
||||
allowed_during_startup = []
|
||||
|
||||
def __init__(self, use_authentication=None):
|
||||
def __init__(self, analytics_manager, use_authentication=None):
|
||||
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
|
||||
self._use_authentication = use_authentication or conf.settings['use_auth_http']
|
||||
self.announced_startup = False
|
||||
self.sessions = {}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_listening(self):
|
||||
from twisted.internet import reactor, error as tx_error
|
||||
|
||||
try:
|
||||
reactor.listenTCP(
|
||||
conf.settings['api_port'], self.get_server_factory(), interface=conf.settings['api_host']
|
||||
)
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
yield self.setup()
|
||||
self.analytics_manager.send_server_startup_success()
|
||||
except tx_error.CannotListenError:
|
||||
log.error('lbrynet API failed to bind TCP %s:%i for listening', conf.settings['api_host'],
|
||||
conf.settings['api_port'])
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except Exception as err:
|
||||
self.analytics_manager.send_server_startup_error(str(err))
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
def setup(self):
|
||||
return NotImplementedError()
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_server_factory(self):
|
||||
return AuthJSONRPCResource(self).getServerFactory()
|
||||
|
||||
def _set_headers(self, request, data, update_secret=False):
|
||||
if conf.settings['allowed_origin']:
|
||||
|
|
@ -233,8 +253,9 @@ class AuthJSONRPCServer(AuthorizedBase):
|
|||
else:
|
||||
# last resort, just cast it as a string
|
||||
error = JSONRPCError(str(failure))
|
||||
log.warning("error processing api request: %s\ntraceback: %s", error.message,
|
||||
"\n".join(error.traceback))
|
||||
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
|
||||
log.warning("error processing api request: %s\ntraceback: %s", error.message,
|
||||
"\n".join(error.traceback))
|
||||
response_content = jsonrpc_dumps_pretty(error, id=id_)
|
||||
self._set_headers(request, response_content)
|
||||
request.setResponseCode(200)
|
||||
|
|
@ -330,14 +351,6 @@ class AuthJSONRPCServer(AuthorizedBase):
|
|||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
except NotAllowedDuringStartupError:
|
||||
log.warning('Function not allowed during startup: %s', function_name)
|
||||
self._render_error(
|
||||
JSONRPCError("This method is unavailable until the daemon is fully started",
|
||||
code=JSONRPCError.CODE_INVALID_REQUEST),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
if args == EMPTY_PARAMS or args == []:
|
||||
_args, _kwargs = (), {}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ class AuthJSONRPCServerTest(unittest.TestCase):
|
|||
# onto it.
|
||||
def setUp(self):
|
||||
conf.initialize_settings(False)
|
||||
self.server = server.AuthJSONRPCServer(use_authentication=False)
|
||||
self.server = server.AuthJSONRPCServer(True, use_authentication=False)
|
||||
|
||||
def test_get_server_port(self):
|
||||
self.assertSequenceEqual(
|
||||
|
|
|
|||
Loading…
Reference in a new issue