diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 9f46406e4..9a640788c 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -1,12 +1,19 @@ import copy +import logging import os import sys + from appdirs import user_data_dir + +log = logging.getLogger(__name__) + + LINUX = 1 DARWIN = 2 WINDOWS = 3 + if sys.platform.startswith("darwin"): platform = DARWIN default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads') @@ -28,6 +35,16 @@ else: def convert_setting(env_val, current_val): + try: + return _convert_setting(env_val, current_val) + except Exception as exc: + log.warning( + 'Failed to convert %s. Returning original: %s: %s', + env_val, current_val, exc) + return current_val + + +def _convert_setting(env_val, current_val): new_type = env_val.__class__ current_type = current_val.__class__ if current_type is bool: @@ -38,7 +55,7 @@ def convert_setting(env_val, current_val): elif str(env_val).lower() == "true": return True else: - raise ValueError + raise ValueError('{} is not a valid boolean value'.format(env_val)) elif current_type is int: return int(env_val) elif current_type is float: @@ -77,6 +94,7 @@ def add_env_settings_to_dict(settings_dict): class Setting(object): + """A collection of configuration settings""" __fixed = [] __excluded = ['get_dict', 'update'] @@ -92,8 +110,9 @@ class Setting(object): def __setitem__(self, key, value): assert key in self and key not in self.__fixed, KeyError(key) - _value = convert_setting(value, self[key]) - self.__dict__.update({key: _value}) + old_value = self[key] + new_value = convert_setting(value, old_value) + self.__dict__[key] = new_value def __contains__(self, item): return item in iter(self) @@ -105,13 +124,12 @@ class Setting(object): for k, v in other.iteritems(): try: self.__setitem__(k, v) - except KeyError: - pass - except AssertionError: + except (KeyError, AssertionError): pass class AdjustableSettings(Setting): + """Settings that are allowed to be overriden by the user""" def __init__(self): self.is_generous_host = True self.run_on_startup = False @@ -161,6 +179,7 @@ class AdjustableSettings(Setting): class ApplicationSettings(Setting): + """Settings that are constants and shouldn't be overriden""" def __init__(self): self.MAX_HANDSHAKE_SIZE = 2**16 self.MAX_REQUEST_SIZE = 2**16 @@ -228,6 +247,24 @@ class Config(DefaultSettings): return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port) +def get_data_dir(): + if sys.platform != "darwin": + data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") + else: + data_dir = user_data_dir("LBRY") + if not os.path.isdir(data_dir): + os.mkdir(data_dir) + return data_dir + + +def get_log_filename(): + """Return the log file for this platform. + + Also ensure the containing directory exists + """ + return os.path.join(get_data_dir(), settings.LOG_FILE_NAME) + + # TODO: don't load the configuration automatically. The configuration # should be loaded at runtime, not at module import time. Module # import should have no side-effects. This is also bad because diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index 8714a30fe..ac4ffcb7a 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -28,11 +28,9 @@ class KeyFeeAboveMaxAllowed(Exception): class UnknownNameError(Exception): def __init__(self, name): + Exception.__init__('Name {} is unknown'.format(name)) self.name = name - def __str__(self): - return repr(self.name) - class InvalidNameError(Exception): def __init__(self, name): diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 1d00e9b13..883e80fc5 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -1377,7 +1377,8 @@ class LBRYumWallet(Wallet): func = getattr(self.cmd_runner, cmd.name) d = threads.deferToThread(func, raw_tx) d.addCallback(_log_tx) - d.addCallback(lambda r: r if len(r) == 64 else defer.fail(Exception("Transaction rejected"))) + d.addCallback( + lambda r: r if len(r) == 64 else defer.fail(Exception("Transaction rejected"))) d.addCallback(self._save_wallet) return d diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 5c23d536a..3d38bc42a 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -1,15 +1,20 @@ +import datetime import json import logging import logging.handlers +import os +import platform import sys import traceback +import requests from requests_futures.sessions import FuturesSession import lbrynet from lbrynet.conf import settings from lbrynet.core import utils + session = FuturesSession() @@ -54,6 +59,12 @@ def remove_handlers(log, handler_name): def _log_decorator(fn): + """Configure a logging handler. + + `fn` is a function that returns a logging handler. The returned + handler has its log-level set and is attached to the specified + logger or the root logger. + """ def helper(*args, **kwargs): log = kwargs.pop('log', logging.getLogger()) level = kwargs.pop('level', logging.INFO) @@ -66,8 +77,12 @@ def _log_decorator(fn): remove_handlers(log, handler.name) handler.setLevel(level) log.addHandler(handler) + # need to reduce the logger's level down to the + # handler's level or else the handler won't + # get those messages if log.level > level: log.setLevel(level) + return handler return helper @@ -76,13 +91,10 @@ def disable_third_party_loggers(): logging.getLogger('urllib3').setLevel(logging.WARNING) logging.getLogger('BitcoinRPC').setLevel(logging.INFO) -def disable_noisy_loggers(): - logging.getLogger('lbrynet').setLevel(logging.INFO) - @_log_decorator def configure_console(**kwargs): - """Convenience function to configure a logger that outputs to stdout""" + """Convenience function to configure a log-handler that outputs to stdout""" handler = logging.StreamHandler(sys.stdout) handler.setFormatter(DEFAULT_FORMATTER) handler.name = 'console' @@ -91,6 +103,7 @@ def configure_console(**kwargs): @_log_decorator def configure_file_handler(file_name, **kwargs): + """Convenience function to configure a log-handler that writes to `file_name`""" handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5) handler.setFormatter(DEFAULT_FORMATTER) handler.name = 'file' @@ -146,4 +159,122 @@ def failure(failure, log, msg, *args): args: values to substitute into `msg` """ args += (failure.getErrorMessage(),) - log.error(msg, *args, exc_info=failure.getTracebackObject()) + exc_info = (failure.type, failure.value, failure.getTracebackObject()) + log.error(msg, *args, exc_info=exc_info) + + +def convert_verbose(verbose): + """Convert the results of the --verbose flag into a list of logger names + + if --verbose is not provided, args.verbose will be None and logging + should be at the info level. + if --verbose is provided, but not followed by any arguments, then + args.verbose = [] and debug logging should be enabled for all of lbrynet + if --verbose is provided and followed by arguments, those arguments + will be in a list + """ + if verbose is None: + return [] + if verbose == []: + return ['lbrynet'] + return verbose + + +def configure_logging(file_name, console, verbose=None): + """Apply the default logging configuration. + + Enables two log-handlers at the INFO level: a file logger and a loggly logger. + Optionally turns on a console logger that defaults to the INFO level, with + specified loggers being set to the DEBUG level. + + Args: + file_name: the file to which logs should be saved + console: If true, enable a console logger + verbose: a list of loggers to set to debug level. + See `convert_verbose` for more details. + """ + verbose = convert_verbose(verbose) + configure_file_handler(file_name) + configure_loggly_handler() + disable_third_party_loggers() + if console: + # if there are some loggers at the debug level, we need + # to enable the console to allow debug. Otherwise, only + # allow info. + level = 'DEBUG' if verbose else 'INFO' + handler = configure_console(level=level) + if verbose: + handler.addFilter(LoggerNameFilter(verbose)) + + +class LoggerNameFilter(object): + """Filter a log record based on its name. + + Allows all info level and higher records to pass thru. + Debug records pass if the log record name (or a parent) match + the input list of logger names. + """ + def __init__(self, logger_names): + self.logger_names = logger_names + + def filter(self, record): + if record.levelno >= logging.INFO: + return True + name = record.name + while name: + if name in self.logger_names: + return True + name = get_parent(name) + return False + + +def get_parent(logger_name): + names = logger_name.split('.') + if len(names) == 1: + return '' + names = names[:-1] + return '.'.join(names) + + +class LogUploader(object): + def __init__(self, log_name, log_file, log_size): + self.log_name = log_name + self.log_file = log_file + self.log_size = log_size + + def upload(self, exclude_previous, id_hash, log_type): + if not os.path.isfile(self.log_file): + return + log_contents = self.log_contents(exclude_previous) + params = { + 'date': datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S'), + 'hash': id_hash, + 'sys': platform.system(), + 'type': self.get_type(log_type), + 'log': log_contents + } + requests.post(settings.LOG_POST_URL, params) + + def log_contents(self, exclude_previous): + with open(self.log_file) as f: + if exclude_previous: + f.seek(self.log_size) + log_contents = f.read() + else: + log_contents = f.read() + return log_contents + + def get_type(self, log_type): + if log_type: + return "%s-%s" % (self.log_name, log_type) + else: + return self.log_name + + @classmethod + def load(cls, log_name, log_file): + if os.path.isfile(log_file): + with open(log_file, 'r') as f: + log_size = len(f.read()) + else: + log_size = 0 + return cls(log_name, log_file, log_size) diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index ad88d3fe4..7858e90f7 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -6,6 +6,7 @@ import json import random import os import socket +import sys import yaml from lbrynet.conf import settings @@ -119,3 +120,9 @@ def check_connection(server="www.lbry.io", port=80): "Failed to connect to %s:%s. Maybe the internet connection is not working", server, port, exc_info=True) return False + + +def setup_certs_for_windows(): + if getattr(sys, 'frozen', False) and os.name == "nt": + cert_path = os.path.join(os.path.dirname(sys.executable), "cacert.pem") + os.environ["REQUESTS_CA_BUNDLE"] = cert_path diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index bb0f0a9cf..b07b22842 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -28,6 +28,7 @@ from lbryum.version import LBRYUM_VERSION as lbryum_version from lbrynet import __version__ as lbrynet_version from lbrynet.conf import settings as lbrynet_settings from lbrynet import analytics +from lbrynet import conf from lbrynet import reflector from lbrynet.metadata.Metadata import Metadata, verify_name_characters from lbrynet.metadata.Fee import FeeValidator @@ -39,7 +40,7 @@ from lbrynet.core.Session import Session from lbrynet.core.looping_call_manager import LoopingCallManager from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory -from lbrynet.core.Error import UnknownNameError, InsufficientFundsError, InvalidNameError +from lbrynet.core.Error import InsufficientFundsError, InvalidNameError from lbrynet.core.PTCWallet import PTCWallet from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet from lbrynet.lbrynet_console.Settings import Settings @@ -56,26 +57,23 @@ from lbrynet.lbrynet_daemon.ExchangeRateManager import ExchangeRateManager from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient from lbrynet.lbrynet_daemon.auth.server import AuthJSONRPCServer +from lbrynet.metadata.Metadata import Metadata, verify_name_characters +from lbrynet.core import log_support +from lbrynet.core import utils +from lbrynet.core.utils import generate_id +from lbrynet.lbrynet_console.Settings import Settings +from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader +from lbrynet.core.Session import Session +from lbrynet.core.PTCWallet import PTCWallet +from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet +from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager +from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager, TempEncryptedFileMetadataManager +from lbrynet import reflector -# TODO: this code snippet is everywhere. Make it go away -if sys.platform != "darwin": - log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -else: - log_dir = user_data_dir("LBRY") - -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -lbrynet_log = os.path.join(log_dir, lbrynet_settings.LOG_FILE_NAME) log = logging.getLogger(__name__) -if os.path.isfile(lbrynet_log): - with open(lbrynet_log, 'r') as f: - PREVIOUS_NET_LOG = len(f.read()) -else: - PREVIOUS_NET_LOG = 0 INITIALIZING_CODE = 'initializing' LOADING_DB_CODE = 'loading_db' @@ -275,12 +273,16 @@ class Daemon(AuthJSONRPCServer): self.ui_version = None self.ip = None self.first_run = None - self.log_file = lbrynet_log + self.log_file = conf.get_log_filename() self.current_db_revision = 1 self.session = None self.first_run_after_update = False self.uploaded_temp_files = [] self._session_id = base58.b58encode(generate_id()) + # TODO: this should probably be passed into the daemon, or + # possibly have the entire log upload functionality taken out + # of the daemon, but I don't want to deal with that now + self.log_uploader = log_support.LogUploader.load('lbrynet', conf.get_log_filename()) self.analytics_manager = None self.lbryid = PENDING_LBRY_ID @@ -324,7 +326,6 @@ class Daemon(AuthJSONRPCServer): self.lbry_file_metadata_manager = None self.lbry_file_manager = None - @AuthJSONRPCServer.subhandler def _exclude_lbrycrd_only_commands_from_lbryum_session(self, request): request.content.seek(0, 0) @@ -400,7 +401,6 @@ class Daemon(AuthJSONRPCServer): self.exchange_rate_manager.start() d = defer.Deferred() - if lbrynet_settings.host_ui: self.lbry_ui_manager.update_checker.start(1800, now=False) d.addCallback(lambda _: self.lbry_ui_manager.setup()) @@ -621,37 +621,20 @@ class Daemon(AuthJSONRPCServer): ds = [] for handler in query_handlers: - ds.append(self.settings.get_query_handler_status(handler.get_primary_query_identifier())) + query_id = handler.get_primary_query_identifier() + ds.append(self.settings.get_query_handler_status(query_id)) dl = defer.DeferredList(ds) dl.addCallback(_set_query_handlers) return dl def _upload_log(self, log_type=None, exclude_previous=False, force=False): if self.upload_log or force: - for lm, lp in [('lbrynet', lbrynet_log)]: - if os.path.isfile(lp): - if exclude_previous: - with open( lp, "r") as f: - f.seek(PREVIOUS_NET_LOG) - log_contents = f.read() - else: - with open(lp, "r") as f: - log_contents = f.read() - if self.lbryid is not PENDING_LBRY_ID: - id_hash = base58.b58encode(self.lbryid)[:20] - else: - id_hash = self.lbryid - params = { - 'date': datetime.utcnow().strftime('%Y%m%d-%H%M%S'), - 'hash': id_hash, - 'sys': platform.system(), - 'type': "%s-%s" % (lm, log_type) if log_type else lm, - 'log': log_contents - } - requests.post(lbrynet_settings.LOG_POST_URL, params) - return defer.succeed(None) - else: - return defer.succeed(None) + if self.lbryid is not PENDING_LBRY_ID: + id_hash = base58.b58encode(self.lbryid)[:20] + else: + id_hash = self.lbryid + self.log_uploader.upload(exclude_previous, self.lbryid, log_type) + return defer.succeed(None) def _clean_up_temp_files(self): for path in self.uploaded_temp_files: @@ -774,13 +757,13 @@ class Daemon(AuthJSONRPCServer): return d def _set_lbryid(self, lbryid): - if lbryid is PENDING_LBRY_ID: - return self._make_lbryid() + if lbryid is PENDING_LBRY_ID or lbryid is None: + return self._make_set_and_save_lbryid() else: log.info("LBRY ID: " + base58.b58encode(lbryid)) self.lbryid = lbryid - def _make_lbryid(self): + def _make_set_and_save_lbryid(self): self.lbryid = generate_id() log.info("Generated new LBRY ID: " + base58.b58encode(self.lbryid)) d = self.settings.save_lbryid(self.lbryid) @@ -1187,7 +1170,9 @@ class Daemon(AuthJSONRPCServer): except: d = defer.fail(None) - d.addCallbacks(lambda r: self._render_response(r, OK_CODE), lambda _: self._render_response(None, OK_CODE)) + d.addCallbacks( + lambda r: self._render_response(r, OK_CODE), + lambda _: self._render_response(None, OK_CODE)) return d @@ -1440,7 +1425,10 @@ class Daemon(AuthJSONRPCServer): return self._render_response(None, BAD_REQUEST) d = self._resolve_name(name, force_refresh=force) - d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) + d.addCallbacks( + lambda info: self._render_response(info, OK_CODE), + errback=handle_failure, errbackArgs=('Failed to resolve name: %s',) + ) return d @AuthJSONRPCServer.auth_required @@ -1531,7 +1519,9 @@ class Daemon(AuthJSONRPCServer): file_name=params.file_name, wait_for_write=params.wait_for_write) # TODO: downloading can timeout. Not sure what to do when that happens - d.addCallbacks(get_output_callback(params), lambda err: str(err)) + d.addCallbacks( + get_output_callback(params), + lambda err: str(err)) d.addCallback(lambda message: self._render_response(message, OK_CODE)) return d @@ -2187,7 +2177,8 @@ class Daemon(AuthJSONRPCServer): check_require = True if 'path' in p: - d = self.lbry_ui_manager.setup(user_specified=p['path'], check_requirements=check_require) + d = self.lbry_ui_manager.setup( + user_specified=p['path'], check_requirements=check_require) elif 'branch' in p: d = self.lbry_ui_manager.setup(branch=p['branch'], check_requirements=check_require) else: @@ -2260,7 +2251,9 @@ class Daemon(AuthJSONRPCServer): sd_hash = p[FileID.SD_HASH] d = self._get_lbry_file(FileID.SD_HASH, sd_hash, return_json=False) d.addCallback(self._reflect) - d.addCallbacks(lambda _: self._render_response(True, OK_CODE), lambda err: self._render_response(err.getTraceback(), OK_CODE)) + d.addCallbacks( + lambda _: self._render_response(True, OK_CODE), + lambda err: self._render_response(err.getTraceback(), OK_CODE)) return d def jsonrpc_get_blob_hashes(self): @@ -2343,8 +2336,9 @@ class Daemon(AuthJSONRPCServer): d = self._resolve_name(name, force_refresh=True) d.addCallback(get_sd_hash) d.addCallback(self._download_sd_blob) - d.addCallbacks(lambda descriptor: [blob.get('blob_hash') for blob in descriptor['blobs']], - lambda _: []) + d.addCallbacks( + lambda descriptor: [blob.get('blob_hash') for blob in descriptor['blobs']], + lambda _: []) d.addCallback(self.session.blob_tracker.get_availability_for_blobs) d.addCallback(_get_mean) d.addCallback(lambda result: self._render_response(result, OK_CODE)) @@ -2541,7 +2535,7 @@ class _ResolveNameHelper(object): if self.need_fresh_stream(): log.info("Resolving stream info for lbry://%s", self.name) d = self.wallet.get_stream_info_for_name(self.name) - d.addCallbacks(self._cache_stream_info, lambda _: defer.fail(UnknownNameError)) + d.addCallback(self._cache_stream_info) else: log.debug("Returning cached stream info for lbry://%s", self.name) d = defer.succeed(self.name_data['claim_metadata']) @@ -2709,3 +2703,15 @@ def get_lbry_file_search_value(p): if value: return searchtype, value raise NoValidSearch() + + +def handle_failure(err, msg): + log_support.failure(err, log, msg) + # TODO: Is this a module? It looks like it: + # + # In [1]: import twisted.web.server + # In [2]: twisted.web.server.failure + # Out[2]: + # + # If so, maybe we should return something else. + return server.failure diff --git a/lbrynet/lbrynet_daemon/DaemonControl.py b/lbrynet/lbrynet_daemon/DaemonControl.py index 474110ab5..74fd6831f 100644 --- a/lbrynet/lbrynet_daemon/DaemonControl.py +++ b/lbrynet/lbrynet_daemon/DaemonControl.py @@ -5,32 +5,23 @@ import webbrowser import sys from twisted.web import server, guard -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, error from twisted.cred import portal - from jsonrpc.proxy import JSONRPCProxy from lbrynet.lbrynet_daemon.auth.auth import PasswordChecker, HttpPasswordRealm from lbrynet.lbrynet_daemon.auth.util import initialize_api_key_file +from lbrynet import conf from lbrynet.core import log_support from lbrynet.core import utils from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest from lbrynet.conf import settings -log_dir = settings.data_dir -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) -if getattr(sys, 'frozen', False) and os.name == "nt": - os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(os.path.dirname(sys.executable), "cacert.pem") - - def test_internet_connection(): return utils.check_connection() @@ -56,45 +47,30 @@ def start(): help="lbrycrd or lbryum, default lbryum", type=str, default='lbryum') - - parser.add_argument("--ui", - help="path to custom UI folder", - default=None) - - parser.add_argument("--branch", - help="Branch of lbry-web-ui repo to use, defaults on master", - default=settings.ui_branch) - - parser.add_argument("--http-auth", - dest="useauth", - action="store_true") - - parser.add_argument('--no-launch', - dest='launchui', - action="store_false") - - parser.add_argument('--log-to-console', - dest='logtoconsole', - action="store_true") - - parser.add_argument('--quiet', - dest='quiet', - action="store_true") - - parser.add_argument('--verbose', - action='store_true', - help='enable more debug output for the console') - - parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False, useauth=settings.use_auth_http) + parser.add_argument("--ui", help="path to custom UI folder", default=None) + parser.add_argument( + "--branch", + help='Branch of lbry-web-ui repo to use, defaults to {}'.format(settings.ui_branch), + default=settings.ui_branch) + parser.add_argument('--no-launch', dest='launchui', action="store_false") + parser.add_argument("--http-auth", dest="useauth", action="store_true") + parser.add_argument( + '--log-to-console', dest='logtoconsole', action='store_true', + help=('Set to enable console logging. Set the --verbose flag ' + ' to enable more detailed console logging')) + parser.add_argument( + '--quiet', dest='quiet', action="store_true", + help=('If log-to-console is not set, setting this disables all console output. ' + 'If log-to-console is set, this argument is ignored')) + parser.add_argument( + '--verbose', nargs="*", + help=('Enable debug output. Optionally specify loggers for which debug output ' + 'should selectively be applied.')) args = parser.parse_args() - log_support.configure_file_handler(lbrynet_log) - log_support.configure_loggly_handler() - if args.logtoconsole: - log_support.configure_console(level='DEBUG') - log_support.disable_third_party_loggers() - if not args.verbose: - log_support.disable_noisy_loggers() + utils.setup_certs_for_windows() + lbrynet_log = conf.get_log_filename() + log_support.configure_logging(lbrynet_log, args.logtoconsole, args.verbose) to_pass = {} settings_path = os.path.join(settings.data_dir, "daemon_settings.yml") @@ -106,9 +82,10 @@ def start(): if args.branch: to_pass.update({'ui_branch': args.branch}) to_pass.update({'use_auth_http': args.useauth}) - to_pass.update({'wallet': args.wallet}) - print to_pass + to_pass.update({'wallet_type': args.wallet}) + log.debug('Settings overrides: %s', to_pass) settings.update(to_pass) + log.debug('Final Settings: %s', settings.__dict__) try: JSONRPCProxy.from_url(settings.API_CONNECTION_STRING).is_running() @@ -131,29 +108,7 @@ def start(): print "To quit press ctrl-c or call 'stop' via the API" if test_internet_connection(): - lbry = DaemonServer() - - d = lbry.start() - if args.launchui: - d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS)) - d.addErrback(log_and_kill) - - if settings.use_auth_http: - log.info("Using authenticated API") - pw_path = os.path.join(settings.data_dir, ".api_keys") - initialize_api_key_file(pw_path) - checker = PasswordChecker.load_file(pw_path) - realm = HttpPasswordRealm(lbry.root) - portal_to_realm = portal.Portal(realm, [checker, ]) - factory = guard.BasicCredentialFactory('Login to lbrynet api') - _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) - else: - log.info("Using non-authenticated API") - _lbrynet_server = server.Site(lbry.root) - - lbrynet_server = server.Site(_lbrynet_server) - lbrynet_server.requestFactory = DaemonRequest - reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE) + start_server_and_listen(args.launchui, args.useauth) reactor.run() if not args.logtoconsole and not args.quiet: @@ -170,5 +125,50 @@ def log_and_kill(failure): reactor.stop() +def start_server_and_listen(launchui, use_auth, **kwargs): + """The primary entry point for launching the daemon. + + Args: + launchui: set to true to open a browser window + use_auth: set to true to enable http authentication + kwargs: passed along to `DaemonServer().start()` + """ + lbry = DaemonServer() + + d = lbry.start(**kwargs) + if launchui: + d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS)) + d.addErrback(log_and_kill) + + site_base = get_site_base(use_auth, lbry.root) + lbrynet_server = server.Site(site_base) + lbrynet_server.requestFactory = DaemonRequest + try: + reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE) + except error.CannotListenError: + log.info('Daemon already running, exiting app') + sys.exit(1) + + +def get_site_base(use_auth, root): + if use_auth: + log.info("Using authenticated API") + return create_auth_session(root) + else: + log.info("Using non-authenticated API") + return server.Site(root) + + +def create_auth_session(root): + pw_path = os.path.join(settings.data_dir, ".api_keys") + initialize_api_key_file(pw_path) + checker = PasswordChecker.load_file(pw_path) + realm = HttpPasswordRealm(root) + portal_to_realm = portal.Portal(realm, [checker, ]) + factory = guard.BasicCredentialFactory('Login to lbrynet api') + _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) + return _lbrynet_server + + if __name__ == "__main__": start() diff --git a/lbrynet/lbrynet_daemon/DaemonServer.py b/lbrynet/lbrynet_daemon/DaemonServer.py index 0980ece2c..94a88f47b 100644 --- a/lbrynet/lbrynet_daemon/DaemonServer.py +++ b/lbrynet/lbrynet_daemon/DaemonServer.py @@ -1,29 +1,21 @@ import logging import os -import sys -from appdirs import user_data_dir from twisted.internet import defer + +from lbrynet import conf from lbrynet.lbrynet_daemon.Daemon import Daemon from lbrynet.lbrynet_daemon.Resources import LBRYindex, HostedEncryptedFile, EncryptedFileUpload from lbrynet.conf import settings -# TODO: omg, this code is essentially duplicated in Daemon -if sys.platform != "darwin": - data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -else: - data_dir = user_data_dir("LBRY") -if not os.path.isdir(data_dir): - os.mkdir(data_dir) - -lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) class DaemonServer(object): def _setup_server(self): - self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active")) + ui_path = os.path.join(conf.get_data_dir(), "lbry-ui", "active") + self.root = LBRYindex(ui_path) self._api = Daemon(self.root) self.root.putChild("view", HostedEncryptedFile(self._api)) self.root.putChild("upload", EncryptedFileUpload(self._api)) diff --git a/lbrynet/lbrynet_daemon/Downloader.py b/lbrynet/lbrynet_daemon/Downloader.py index 5d7c364d7..97446f2ef 100644 --- a/lbrynet/lbrynet_daemon/Downloader.py +++ b/lbrynet/lbrynet_daemon/Downloader.py @@ -1,10 +1,8 @@ import json import logging import os -import sys from copy import deepcopy -from appdirs import user_data_dir from twisted.internet import defer from twisted.internet.task import LoopingCall @@ -20,28 +18,24 @@ DOWNLOAD_TIMEOUT_CODE = 'timeout' DOWNLOAD_RUNNING_CODE = 'running' DOWNLOAD_STOPPED_CODE = 'stopped' STREAM_STAGES = [ - (INITIALIZING_CODE, 'Initializing...'), - (DOWNLOAD_METADATA_CODE, 'Downloading metadata'), - (DOWNLOAD_RUNNING_CODE, 'Started stream'), - (DOWNLOAD_STOPPED_CODE, 'Paused stream'), - (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out') - ] + (INITIALIZING_CODE, 'Initializing...'), + (DOWNLOAD_METADATA_CODE, 'Downloading metadata'), + (DOWNLOAD_RUNNING_CODE, 'Started stream'), + (DOWNLOAD_STOPPED_CODE, 'Paused stream'), + (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out') +] -if sys.platform != "darwin": - log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -else: - log_dir = user_data_dir("LBRY") -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) class GetStream(object): - def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager, - max_key_fee, data_rate=0.5, timeout=settings.download_timeout, download_directory=None, file_name=None): + def __init__(self, sd_identifier, session, wallet, + lbry_file_manager, exchange_rate_manager, + max_key_fee, data_rate=0.5, timeout=None, + download_directory=None, file_name=None): + if timeout is None: + timeout = settings.download_timeout self.wallet = wallet self.resolved_name = None self.description = None diff --git a/lbrynet/lbrynet_daemon/Publisher.py b/lbrynet/lbrynet_daemon/Publisher.py index 8cd956d42..98654d5e8 100644 --- a/lbrynet/lbrynet_daemon/Publisher.py +++ b/lbrynet/lbrynet_daemon/Publisher.py @@ -1,10 +1,9 @@ import logging import mimetypes import os -import sys import random -from appdirs import user_data_dir +from twisted.internet import threads, defer, reactor from lbrynet.core.Error import InsufficientFundsError from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file @@ -13,17 +12,8 @@ from lbrynet.metadata.Metadata import Metadata from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet import reflector from lbrynet.conf import settings -from twisted.internet import threads, defer, reactor -if sys.platform != "darwin": - log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -else: - log_dir = user_data_dir("LBRY") -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) diff --git a/lbrynet/lbrynet_daemon/UIManager.py b/lbrynet/lbrynet_daemon/UIManager.py index c0e524eab..2169bdaea 100644 --- a/lbrynet/lbrynet_daemon/UIManager.py +++ b/lbrynet/lbrynet_daemon/UIManager.py @@ -15,15 +15,7 @@ from lbryum.version import LBRYUM_VERSION as lbryum_version from zipfile import ZipFile from appdirs import user_data_dir -if sys.platform != "darwin": - log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -else: - log_dir = user_data_dir("LBRY") -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) diff --git a/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py b/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py index ab43236f3..5646d67dd 100644 --- a/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py +++ b/packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py @@ -55,7 +55,6 @@ class LBRYDaemonApp(AppKit.NSApplication): self.statusitem.setMenu_(self.menubarMenu) self.statusitem.setToolTip_(settings.APP_NAME) - if test_internet_connection(): if platform.mac_ver()[0] >= "10.10": LBRYNotify("Starting LBRY") @@ -64,12 +63,7 @@ class LBRYDaemonApp(AppKit.NSApplication): LBRYNotify("LBRY needs an internet connection to start, try again when one is available") sys.exit(0) - lbry = DaemonServer() - d = lbry.start(use_authentication=False) - d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS)) - lbrynet_server = server.Site(lbry.root) - lbrynet_server.requestFactory = DaemonRequest - reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE) + DaemonControl.start_server_and_listen(launchui=True, use_auth=False) def openui_(self, sender): webbrowser.open(settings.UI_ADDRESS) diff --git a/packaging/osx/lbry-osx-app/lbrygui/main.py b/packaging/osx/lbry-osx-app/lbrygui/main.py index d7ca4dc00..ce3e22631 100644 --- a/packaging/osx/lbry-osx-app/lbrygui/main.py +++ b/packaging/osx/lbry-osx-app/lbrygui/main.py @@ -4,32 +4,22 @@ install(runner=AppHelper.runEventLoop) from twisted.internet import reactor import logging -import sys -import os -from appdirs import user_data_dir +from lbrynet import conf +from lbrynet.core import log_support from LBRYApp import LBRYDaemonApp -if sys.platform != "darwin": - log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -else: - log_dir = user_data_dir("LBRY") -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log') - -log = logging.getLogger(__name__) -handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5) -log.addHandler(handler) -logging.basicConfig(level=logging.INFO) +log = logging.getLogger() def main(): + log_file = conf.get_log_filename() + log_support.configure_logging(log_file, console=False) app = LBRYDaemonApp.sharedApplication() reactor.addSystemEventTrigger("after", "shutdown", AppHelper.stopEventLoop) reactor.run() + if __name__ == "__main__": main() diff --git a/packaging/windows/lbry-win32-app/LBRYWin32App.py b/packaging/windows/lbry-win32-app/LBRYWin32App.py index d3591dcab..0f637d0d8 100644 --- a/packaging/windows/lbry-win32-app/LBRYWin32App.py +++ b/packaging/windows/lbry-win32-app/LBRYWin32App.py @@ -1,6 +1,5 @@ import logging import os -import socket import sys import threading import webbrowser @@ -11,31 +10,24 @@ import win32gui_struct from jsonrpc.proxy import JSONRPCProxy from twisted.internet import reactor, error from twisted.web import server -import twisted try: import winxpgui as win32gui except ImportError: import win32gui +from lbrynet import conf +from lbrynet.core import log_support from lbrynet.core import utils +from lbrynet.lbrynet_daemon import DaemonControl from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest from lbrynet.conf import settings from packaging.uri_handler.LBRYURIHandler import LBRYURIHandler -# TODO: omg, this code is essentially duplicated in LBRYDaemon -data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") -if not os.path.isdir(data_dir): - os.mkdir(data_dir) - -lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) -if getattr(sys, 'frozen', False) and os.name == "nt": - os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(os.path.dirname(sys.executable), "cacert.pem") - def test_internet_connection(): return utils.check_connection() @@ -278,19 +270,14 @@ def main(lbry_name=None): systray_thread.daemon = True systray_thread.start() - lbry = DaemonServer() - d = lbry.start(use_authentication=False) - d.addCallback(lambda _: LBRYURIHandler.open_address(lbry_name)) - lbrynet_server = server.Site(lbry.root) - lbrynet_server.requestFactory = DaemonRequest - try: - reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE) - except error.CannotListenError: - log.info('Daemon already running, exiting app') - sys.exit(1) + DaemonControl.start_server_and_listen(launchui=True, use_auth=False) reactor.run() + if __name__ == '__main__': + utils.setup_certs_for_windows() + log_file = conf.get_log_filename() + log_support.configure_logging(log_file, console=False) lbry_daemon = JSONRPCProxy.from_url(settings.API_CONNECTION_STRING) try: