From 2ee6fac01436fde0acc26d57452fbd9e02dd415f Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 21 Oct 2016 15:26:36 -0700 Subject: [PATCH] redo logging configuration for lbrynet-daemon script --- lbrynet/core/log_support.py | 117 ++++++++++++++++++++++-- lbrynet/core/utils.py | 7 ++ lbrynet/lbrynet_daemon/DaemonControl.py | 70 +++++--------- 3 files changed, 136 insertions(+), 58 deletions(-) diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 46ce6e4e0..635542480 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -1,15 +1,18 @@ import json import logging import logging.handlers +import os import sys import traceback +import appdirs from requests_futures.sessions import FuturesSession import lbrynet from lbrynet.conf import settings from lbrynet.core import utils + session = FuturesSession() @@ -54,6 +57,12 @@ def remove_handlers(log, handler_name): def _log_decorator(fn): + """Configure a logging handler. + + `fn` is a function that returns a logging handler. The returned + handler has its log-level set and is attached to the specified + logger or the root logger. + """ def helper(*args, **kwargs): log = kwargs.pop('log', logging.getLogger()) level = kwargs.pop('level', logging.INFO) @@ -66,8 +75,12 @@ def _log_decorator(fn): remove_handlers(log, handler.name) handler.setLevel(level) log.addHandler(handler) + # need to reduce the logger's level down to the + # handler's level or else the handler won't + # get those messages if log.level > level: log.setLevel(level) + return handler return helper @@ -76,19 +89,10 @@ def disable_third_party_loggers(): logging.getLogger('urllib3').setLevel(logging.WARNING) logging.getLogger('BitcoinRPC').setLevel(logging.INFO) -def disable_noisy_loggers(): - logging.getLogger('lbrynet.analytics.api').setLevel(logging.INFO) - logging.getLogger('lbrynet.core').setLevel(logging.INFO) - logging.getLogger('lbrynet.dht').setLevel(logging.INFO) - logging.getLogger('lbrynet.lbrynet_daemon').setLevel(logging.INFO) - logging.getLogger('lbrynet.core.Wallet').setLevel(logging.INFO) - logging.getLogger('lbrynet.lbryfile').setLevel(logging.INFO) - logging.getLogger('lbrynet.lbryfilemanager').setLevel(logging.INFO) - @_log_decorator def configure_console(**kwargs): - """Convenience function to configure a logger that outputs to stdout""" + """Convenience function to configure a log-handler that outputs to stdout""" handler = logging.StreamHandler(sys.stdout) handler.setFormatter(DEFAULT_FORMATTER) handler.name = 'console' @@ -97,6 +101,7 @@ def configure_console(**kwargs): @_log_decorator def configure_file_handler(file_name, **kwargs): + """Convenience function to configure a log-handler that writes to `file_name`""" handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5) handler.setFormatter(DEFAULT_FORMATTER) handler.name = 'file' @@ -153,3 +158,95 @@ def failure(failure, log, msg, *args): """ args += (failure.getErrorMessage(),) log.error(msg, *args, exc_info=failure.getTracebackObject()) + + +def convert_verbose(verbose): + """Convert the results of the --verbose flag into a list of logger names + + if --verbose is not provided, args.verbose will be None and logging + should be at the info level. + if --verbose is provided, but not followed by any arguments, then + args.verbose = [] and debug logging should be enabled for all of lbrynet + if --verbose is provided and followed by arguments, those arguments + will be in a list + """ + if verbose is None: + return [] + if verbose == []: + return ['lbrynet'] + return verbose + + +def configure_logging(file_name, console, verbose=None): + """Apply the default logging configuration. + + Enables two log-handlers at the INFO level: a file logger and a loggly logger. + Optionally turns on a console logger that defaults to the INFO level, with + specified loggers being set to the DEBUG level. + + Args: + file_name: the file to which logs should be saved + console: If true, enable a console logger + verbose: a list of loggers to set to debug level. + See `convert_verbose` for more details. + """ + verbose = convert_verbose(verbose) + configure_file_handler(file_name) + configure_loggly_handler() + disable_third_party_loggers() + if console: + # if there are some loggers at the debug level, we need + # to enable the console to allow debug. Otherwise, only + # allow info. + level = 'DEBUG' if verbose else 'INFO' + handler = configure_console(level=level) + if verbose: + handler.addFilter(LoggerNameFilter(verbose)) + + +def get_log_file(): + """Return the log file for this platform. + + Also ensure the containing directory exists + """ + if sys.platform != "darwin": + log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") + else: + log_dir = appdirs.user_data_dir("LBRY") + + try: + os.mkdir(log_dir) + except OSError: + pass + + lbrynet_log = os.path.join(log_dir, conf.LOG_FILE_NAME) + return lbrynet_log + + +class LoggerNameFilter(object): + """Filter a log record based on its name. + + Allows all info level and higher records to pass thru. + Debug records pass if the log record name (or a parent) match + the input list of logger names. + """ + def __init__(self, logger_names): + self.logger_names = logger_names + + def filter(self, record): + if record.levelno >= logging.INFO: + return True + name = record.name + while name: + if name in self.logger_names: + return True + name = get_parent(name) + return False + + +def get_parent(logger_name): + names = logger_name.split('.') + if len(names) == 1: + return '' + names = names[:-1] + return '.'.join(names) diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index ad88d3fe4..7858e90f7 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -6,6 +6,7 @@ import json import random import os import socket +import sys import yaml from lbrynet.conf import settings @@ -119,3 +120,9 @@ def check_connection(server="www.lbry.io", port=80): "Failed to connect to %s:%s. Maybe the internet connection is not working", server, port, exc_info=True) return False + + +def setup_certs_for_windows(): + if getattr(sys, 'frozen', False) and os.name == "nt": + cert_path = os.path.join(os.path.dirname(sys.executable), "cacert.pem") + os.environ["REQUESTS_CA_BUNDLE"] = cert_path diff --git a/lbrynet/lbrynet_daemon/DaemonControl.py b/lbrynet/lbrynet_daemon/DaemonControl.py index 474110ab5..c6437785c 100644 --- a/lbrynet/lbrynet_daemon/DaemonControl.py +++ b/lbrynet/lbrynet_daemon/DaemonControl.py @@ -1,8 +1,6 @@ import argparse import logging.handlers -import os import webbrowser -import sys from twisted.web import server, guard from twisted.internet import defer, reactor @@ -18,19 +16,10 @@ from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest from lbrynet.conf import settings -log_dir = settings.data_dir -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME) log = logging.getLogger(__name__) -if getattr(sys, 'frozen', False) and os.name == "nt": - os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(os.path.dirname(sys.executable), "cacert.pem") - - def test_internet_connection(): return utils.check_connection() @@ -56,45 +45,30 @@ def start(): help="lbrycrd or lbryum, default lbryum", type=str, default='lbryum') - - parser.add_argument("--ui", - help="path to custom UI folder", - default=None) - - parser.add_argument("--branch", - help="Branch of lbry-web-ui repo to use, defaults on master", - default=settings.ui_branch) - - parser.add_argument("--http-auth", - dest="useauth", - action="store_true") - - parser.add_argument('--no-launch', - dest='launchui', - action="store_false") - - parser.add_argument('--log-to-console', - dest='logtoconsole', - action="store_true") - - parser.add_argument('--quiet', - dest='quiet', - action="store_true") - - parser.add_argument('--verbose', - action='store_true', - help='enable more debug output for the console') - - parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False, useauth=settings.use_auth_http) + parser.add_argument("--ui", help="path to custom UI folder", default=None) + parser.add_argument( + "--branch", + help='Branch of lbry-web-ui repo to use, defaults to {}'.format(settings.ui_branch), + default=settings.ui_branch) + parser.add_argument('--no-launch', dest='launchui', action="store_false") + parser.add_argument("--http-auth", dest="useauth", action="store_true") + parser.add_argument( + '--log-to-console', dest='logtoconsole', action='store_true', + help=('Set to enable console logging. Set the --verbose flag ' + ' to enable more detailed console logging')) + parser.add_argument( + '--quiet', dest='quiet', action="store_true", + help=('If log-to-console is not set, setting this disables all console output. ' + 'If log-to-console is set, this argument is ignored')) + parser.add_argument( + '--verbose', nargs="*", + help=('Enable debug output. Optionally specify loggers for which debug output ' + 'should selectively be applied.')) args = parser.parse_args() - log_support.configure_file_handler(lbrynet_log) - log_support.configure_loggly_handler() - if args.logtoconsole: - log_support.configure_console(level='DEBUG') - log_support.disable_third_party_loggers() - if not args.verbose: - log_support.disable_noisy_loggers() + utils.setup_certs_for_windows() + lbrynet_log = log_support.get_log_file() + log_support.configure_logging(lbrynet_log, args.logtoconsole, args.verbose) to_pass = {} settings_path = os.path.join(settings.data_dir, "daemon_settings.yml")