Merge remote-tracking branch 'origin/master' into reflect-my-unavailable-streams

Conflicts:
	lbrynet/core/log_support.py

lbrynet/conf.py

@@ -1,12 +1,19 @@
 import copy
+import logging
 import os
 import sys
+
 from appdirs import user_data_dir
 
+
+log = logging.getLogger(__name__)
+
+
 LINUX = 1
 DARWIN = 2
 WINDOWS = 3
 
+
 if sys.platform.startswith("darwin"):
     platform = DARWIN
     default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
@@ -28,6 +35,16 @@ else:
 
 
 def convert_setting(env_val, current_val):
+    try:
+        return _convert_setting(env_val, current_val)
+    except Exception as exc:
+        log.warning(
+            'Failed to convert %s. Returning original: %s: %s',
+            env_val, current_val, exc)
+        return current_val
+
+
+def _convert_setting(env_val, current_val):
     new_type = env_val.__class__
     current_type = current_val.__class__
     if current_type is bool:
@@ -38,7 +55,7 @@ def convert_setting(env_val, current_val):
         elif str(env_val).lower() == "true":
             return True
         else:
-            raise ValueError
+            raise ValueError('{} is not a valid boolean value'.format(env_val))
     elif current_type is int:
         return int(env_val)
     elif current_type is float:
@@ -77,6 +94,7 @@ def add_env_settings_to_dict(settings_dict):
 
 
 class Setting(object):
+    """A collection of configuration settings"""
     __fixed = []
     __excluded = ['get_dict', 'update']
 
@@ -92,8 +110,9 @@ class Setting(object):
 
     def __setitem__(self, key, value):
         assert key in self and key not in self.__fixed, KeyError(key)
-        _value = convert_setting(value, self[key])
+        old_value = self[key]
-        self.__dict__.update({key: _value})
+        new_value = convert_setting(value, old_value)
+        self.__dict__[key] = new_value
 
     def __contains__(self, item):
         return item in iter(self)
@@ -105,13 +124,12 @@ class Setting(object):
         for k, v in other.iteritems():
             try:
                 self.__setitem__(k, v)
-            except KeyError:
+            except (KeyError, AssertionError):
-                pass
-            except AssertionError:
                 pass
 
 
 class AdjustableSettings(Setting):
+    """Settings that are allowed to be overriden by the user"""
     def __init__(self):
         self.is_generous_host = True
         self.run_on_startup = False
@@ -161,6 +179,7 @@ class AdjustableSettings(Setting):
 
 
 class ApplicationSettings(Setting):
+    """Settings that are constants and shouldn't be overriden"""
     def __init__(self):
         self.MAX_HANDSHAKE_SIZE = 2**16
         self.MAX_REQUEST_SIZE = 2**16
@@ -228,6 +247,24 @@ class Config(DefaultSettings):
         return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port)
 
 
+def get_data_dir():
+    if sys.platform != "darwin":
+        data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
+    else:
+        data_dir = user_data_dir("LBRY")
+    if not os.path.isdir(data_dir):
+        os.mkdir(data_dir)
+    return data_dir
+
+
+def get_log_filename():
+    """Return the log file for this platform.
+
+    Also ensure the containing directory exists
+    """
+    return os.path.join(get_data_dir(), settings.LOG_FILE_NAME)
+
+
 # TODO: don't load the configuration automatically. The configuration
 #       should be loaded at runtime, not at module import time. Module
 #       import should have no side-effects. This is also bad because

lbrynet/core/Error.py

@@ -28,11 +28,9 @@ class KeyFeeAboveMaxAllowed(Exception):
 
 class UnknownNameError(Exception):
     def __init__(self, name):
+        Exception.__init__('Name {} is unknown'.format(name))
         self.name = name
 
-    def __str__(self):
-        return repr(self.name)
-
 
 class InvalidNameError(Exception):
     def __init__(self, name):

lbrynet/core/Wallet.py

@@ -1377,7 +1377,8 @@ class LBRYumWallet(Wallet):
         func = getattr(self.cmd_runner, cmd.name)
         d = threads.deferToThread(func, raw_tx)
         d.addCallback(_log_tx)
-        d.addCallback(lambda r: r if len(r) == 64 else defer.fail(Exception("Transaction rejected")))
+        d.addCallback(
+            lambda r: r if len(r) == 64 else defer.fail(Exception("Transaction rejected")))
         d.addCallback(self._save_wallet)
         return d
 

lbrynet/core/log_support.py

@@ -1,15 +1,20 @@
+import datetime
 import json
 import logging
 import logging.handlers
+import os
+import platform
 import sys
 import traceback
 
+import requests
 from requests_futures.sessions import FuturesSession
 
 import lbrynet
 from lbrynet.conf import settings
 from lbrynet.core import utils
 
+
 session = FuturesSession()
 
 
@@ -54,6 +59,12 @@ def remove_handlers(log, handler_name):
 
 
 def _log_decorator(fn):
+    """Configure a logging handler.
+
+    `fn` is a function that returns a logging handler. The returned
+    handler has its log-level set and is attached to the specified
+    logger or the root logger.
+    """
     def helper(*args, **kwargs):
         log = kwargs.pop('log', logging.getLogger())
         level = kwargs.pop('level', logging.INFO)
@@ -66,8 +77,12 @@ def _log_decorator(fn):
             remove_handlers(log, handler.name)
         handler.setLevel(level)
         log.addHandler(handler)
+        # need to reduce the logger's level down to the
+        # handler's level or else the handler won't
+        # get those messages
         if log.level > level:
             log.setLevel(level)
+        return handler
     return helper
 
 
@@ -76,13 +91,10 @@ def disable_third_party_loggers():
     logging.getLogger('urllib3').setLevel(logging.WARNING)
     logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
 
-def disable_noisy_loggers():
-    logging.getLogger('lbrynet').setLevel(logging.INFO)
-
 
 @_log_decorator
 def configure_console(**kwargs):
-    """Convenience function to configure a logger that outputs to stdout"""
+    """Convenience function to configure a log-handler that outputs to stdout"""
     handler = logging.StreamHandler(sys.stdout)
     handler.setFormatter(DEFAULT_FORMATTER)
     handler.name = 'console'
@@ -91,6 +103,7 @@ def configure_console(**kwargs):
 
 @_log_decorator
 def configure_file_handler(file_name, **kwargs):
+    """Convenience function to configure a log-handler that writes to `file_name`"""
     handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
     handler.setFormatter(DEFAULT_FORMATTER)
     handler.name = 'file'
@@ -146,4 +159,122 @@ def failure(failure, log, msg, *args):
         args: values to substitute into `msg`
     """
     args += (failure.getErrorMessage(),)
-    log.error(msg, *args, exc_info=failure.getTracebackObject())
+    exc_info = (failure.type, failure.value, failure.getTracebackObject())
+    log.error(msg, *args, exc_info=exc_info)
+
+
+def convert_verbose(verbose):
+    """Convert the results of the --verbose flag into a list of logger names
+
+    if --verbose is not provided, args.verbose will be None and logging
+    should be at the info level.
+    if --verbose is provided, but not followed by any arguments, then
+    args.verbose = [] and debug logging should be enabled for all of lbrynet
+    if --verbose is provided and followed by arguments, those arguments
+    will be in a list
+    """
+    if verbose is None:
+        return []
+    if verbose == []:
+        return ['lbrynet']
+    return verbose
+
+
+def configure_logging(file_name, console, verbose=None):
+    """Apply the default logging configuration.
+
+    Enables two log-handlers at the INFO level: a file logger and a loggly logger.
+    Optionally turns on a console logger that defaults to the INFO level, with
+    specified loggers being set to the DEBUG level.
+
+    Args:
+        file_name: the file to which logs should be saved
+        console: If true, enable a console logger
+        verbose: a list of loggers to set to debug level.
+            See `convert_verbose` for more details.
+    """
+    verbose = convert_verbose(verbose)
+    configure_file_handler(file_name)
+    configure_loggly_handler()
+    disable_third_party_loggers()
+    if console:
+        # if there are some loggers at the debug level, we need
+        # to enable the console to allow debug. Otherwise, only
+        # allow info.
+        level = 'DEBUG' if verbose else 'INFO'
+        handler = configure_console(level=level)
+        if verbose:
+            handler.addFilter(LoggerNameFilter(verbose))
+
+
+class LoggerNameFilter(object):
+    """Filter a log record based on its name.
+
+    Allows all info level and higher records to pass thru.
+    Debug records pass if the log record name (or a parent) match
+    the input list of logger names.
+    """
+    def __init__(self, logger_names):
+        self.logger_names = logger_names
+
+    def filter(self, record):
+        if record.levelno >= logging.INFO:
+            return True
+        name = record.name
+        while name:
+            if name in self.logger_names:
+                return True
+            name = get_parent(name)
+        return False
+
+
+def get_parent(logger_name):
+    names = logger_name.split('.')
+    if len(names) == 1:
+        return ''
+    names = names[:-1]
+    return '.'.join(names)
+
+
+class LogUploader(object):
+    def __init__(self, log_name, log_file, log_size):
+        self.log_name = log_name
+        self.log_file = log_file
+        self.log_size = log_size
+
+    def upload(self, exclude_previous, id_hash, log_type):
+        if not os.path.isfile(self.log_file):
+            return
+        log_contents = self.log_contents(exclude_previous)
+        params = {
+            'date': datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S'),
+            'hash': id_hash,
+            'sys': platform.system(),
+            'type': self.get_type(log_type),
+            'log': log_contents
+        }
+        requests.post(settings.LOG_POST_URL, params)
+
+    def log_contents(self, exclude_previous):
+        with open(self.log_file) as f:
+            if exclude_previous:
+                f.seek(self.log_size)
+                log_contents = f.read()
+            else:
+                log_contents = f.read()
+        return log_contents
+
+    def get_type(self, log_type):
+        if log_type:
+            return "%s-%s" % (self.log_name, log_type)
+        else:
+            return self.log_name
+
+    @classmethod
+    def load(cls, log_name, log_file):
+        if os.path.isfile(log_file):
+            with open(log_file, 'r') as f:
+                log_size = len(f.read())
+        else:
+            log_size = 0
+        return cls(log_name, log_file, log_size)

lbrynet/core/utils.py

@@ -6,6 +6,7 @@ import json
 import random
 import os
 import socket
+import sys
 import yaml
 
 from lbrynet.conf import settings
@@ -119,3 +120,9 @@ def check_connection(server="www.lbry.io", port=80):
             "Failed to connect to %s:%s. Maybe the internet connection is not working",
             server, port, exc_info=True)
         return False
+
+
+def setup_certs_for_windows():
+    if getattr(sys, 'frozen', False) and os.name == "nt":
+        cert_path = os.path.join(os.path.dirname(sys.executable), "cacert.pem")
+        os.environ["REQUESTS_CA_BUNDLE"] = cert_path

lbrynet/lbrynet_daemon/Daemon.py

@@ -28,6 +28,7 @@ from lbryum.version import LBRYUM_VERSION as lbryum_version
 from lbrynet import __version__ as lbrynet_version
 from lbrynet.conf import settings as lbrynet_settings
 from lbrynet import analytics
+from lbrynet import conf
 from lbrynet import reflector
 from lbrynet.metadata.Metadata import Metadata, verify_name_characters
 from lbrynet.metadata.Fee import FeeValidator
@@ -39,7 +40,7 @@ from lbrynet.core.Session import Session
 from lbrynet.core.looping_call_manager import LoopingCallManager
 from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
 from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
-from lbrynet.core.Error import UnknownNameError, InsufficientFundsError, InvalidNameError
+from lbrynet.core.Error import InsufficientFundsError, InvalidNameError
 from lbrynet.core.PTCWallet import PTCWallet
 from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet
 from lbrynet.lbrynet_console.Settings import Settings
@@ -56,26 +57,23 @@ from lbrynet.lbrynet_daemon.ExchangeRateManager import ExchangeRateManager
 from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient
 from lbrynet.lbrynet_daemon.auth.server import AuthJSONRPCServer
 
+from lbrynet.metadata.Metadata import Metadata, verify_name_characters
+from lbrynet.core import log_support
+from lbrynet.core import utils
+from lbrynet.core.utils import generate_id
+from lbrynet.lbrynet_console.Settings import Settings
 
+from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader
+from lbrynet.core.Session import Session
+from lbrynet.core.PTCWallet import PTCWallet
+from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet
+from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
+from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager, TempEncryptedFileMetadataManager
+from lbrynet import reflector
 
-# TODO: this code snippet is everywhere. Make it go away
-if sys.platform != "darwin":
-    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-else:
-    log_dir = user_data_dir("LBRY")
-
-if not os.path.isdir(log_dir):
-    os.mkdir(log_dir)
-
-lbrynet_log = os.path.join(log_dir, lbrynet_settings.LOG_FILE_NAME)
 
 log = logging.getLogger(__name__)
 
-if os.path.isfile(lbrynet_log):
-    with open(lbrynet_log, 'r') as f:
-        PREVIOUS_NET_LOG = len(f.read())
-else:
-    PREVIOUS_NET_LOG = 0
 
 INITIALIZING_CODE = 'initializing'
 LOADING_DB_CODE = 'loading_db'
@@ -275,12 +273,16 @@ class Daemon(AuthJSONRPCServer):
         self.ui_version = None
         self.ip = None
         self.first_run = None
-        self.log_file = lbrynet_log
+        self.log_file = conf.get_log_filename()
         self.current_db_revision = 1
         self.session = None
         self.first_run_after_update = False
         self.uploaded_temp_files = []
         self._session_id = base58.b58encode(generate_id())
+        # TODO: this should probably be passed into the daemon, or
+        # possibly have the entire log upload functionality taken out
+        # of the daemon, but I don't want to deal with that now
+        self.log_uploader = log_support.LogUploader.load('lbrynet', conf.get_log_filename())
 
         self.analytics_manager = None
         self.lbryid = PENDING_LBRY_ID
@@ -324,7 +326,6 @@ class Daemon(AuthJSONRPCServer):
         self.lbry_file_metadata_manager = None
         self.lbry_file_manager = None
 
-
     @AuthJSONRPCServer.subhandler
     def _exclude_lbrycrd_only_commands_from_lbryum_session(self, request):
         request.content.seek(0, 0)
@@ -400,7 +401,6 @@ class Daemon(AuthJSONRPCServer):
         self.exchange_rate_manager.start()
 
         d = defer.Deferred()
-
         if lbrynet_settings.host_ui:
             self.lbry_ui_manager.update_checker.start(1800, now=False)
             d.addCallback(lambda _: self.lbry_ui_manager.setup())
@@ -621,37 +621,20 @@ class Daemon(AuthJSONRPCServer):
 
         ds = []
         for handler in query_handlers:
-            ds.append(self.settings.get_query_handler_status(handler.get_primary_query_identifier()))
+            query_id = handler.get_primary_query_identifier()
+            ds.append(self.settings.get_query_handler_status(query_id))
         dl = defer.DeferredList(ds)
         dl.addCallback(_set_query_handlers)
         return dl
 
     def _upload_log(self, log_type=None, exclude_previous=False, force=False):
         if self.upload_log or force:
-            for lm, lp in [('lbrynet', lbrynet_log)]:
+            if self.lbryid is not PENDING_LBRY_ID:
-                if os.path.isfile(lp):
+                id_hash = base58.b58encode(self.lbryid)[:20]
-                    if exclude_previous:
+            else:
-                        with open( lp, "r") as f:
+                id_hash = self.lbryid
-                            f.seek(PREVIOUS_NET_LOG)
+            self.log_uploader.upload(exclude_previous, self.lbryid, log_type)
-                            log_contents = f.read()
+        return defer.succeed(None)
-                    else:
-                        with open(lp, "r") as f:
-                            log_contents = f.read()
-                    if self.lbryid is not PENDING_LBRY_ID:
-                        id_hash = base58.b58encode(self.lbryid)[:20]
-                    else:
-                        id_hash = self.lbryid
-                    params = {
-                        'date': datetime.utcnow().strftime('%Y%m%d-%H%M%S'),
-                        'hash': id_hash,
-                        'sys': platform.system(),
-                        'type': "%s-%s" % (lm, log_type) if log_type else lm,
-                        'log': log_contents
-                    }
-                    requests.post(lbrynet_settings.LOG_POST_URL, params)
-            return defer.succeed(None)
-        else:
-            return defer.succeed(None)
 
     def _clean_up_temp_files(self):
         for path in self.uploaded_temp_files:
@@ -774,13 +757,13 @@ class Daemon(AuthJSONRPCServer):
         return d
 
     def _set_lbryid(self, lbryid):
-        if lbryid is PENDING_LBRY_ID:
+        if lbryid is PENDING_LBRY_ID or lbryid is None:
-            return self._make_lbryid()
+            return self._make_set_and_save_lbryid()
         else:
             log.info("LBRY ID: " + base58.b58encode(lbryid))
             self.lbryid = lbryid
 
-    def _make_lbryid(self):
+    def _make_set_and_save_lbryid(self):
         self.lbryid = generate_id()
         log.info("Generated new LBRY ID: " + base58.b58encode(self.lbryid))
         d = self.settings.save_lbryid(self.lbryid)
@@ -1187,7 +1170,9 @@ class Daemon(AuthJSONRPCServer):
         except:
             d = defer.fail(None)
 
-        d.addCallbacks(lambda r: self._render_response(r, OK_CODE), lambda _: self._render_response(None, OK_CODE))
+        d.addCallbacks(
+            lambda r: self._render_response(r, OK_CODE),
+            lambda _: self._render_response(None, OK_CODE))
 
         return d
 
@@ -1440,7 +1425,10 @@ class Daemon(AuthJSONRPCServer):
             return self._render_response(None, BAD_REQUEST)
 
         d = self._resolve_name(name, force_refresh=force)
-        d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure)
+        d.addCallbacks(
+            lambda info: self._render_response(info, OK_CODE),
+            errback=handle_failure, errbackArgs=('Failed to resolve name: %s',)
+        )
         return d
 
     @AuthJSONRPCServer.auth_required
@@ -1531,7 +1519,9 @@ class Daemon(AuthJSONRPCServer):
                                 file_name=params.file_name,
                                 wait_for_write=params.wait_for_write)
         # TODO: downloading can timeout.  Not sure what to do when that happens
-        d.addCallbacks(get_output_callback(params), lambda err: str(err))
+        d.addCallbacks(
+            get_output_callback(params),
+            lambda err: str(err))
         d.addCallback(lambda message: self._render_response(message, OK_CODE))
         return d
 
@@ -2187,7 +2177,8 @@ class Daemon(AuthJSONRPCServer):
             check_require = True
 
         if 'path' in p:
-            d = self.lbry_ui_manager.setup(user_specified=p['path'], check_requirements=check_require)
+            d = self.lbry_ui_manager.setup(
+                user_specified=p['path'], check_requirements=check_require)
         elif 'branch' in p:
             d = self.lbry_ui_manager.setup(branch=p['branch'], check_requirements=check_require)
         else:
@@ -2260,7 +2251,9 @@ class Daemon(AuthJSONRPCServer):
         sd_hash = p[FileID.SD_HASH]
         d = self._get_lbry_file(FileID.SD_HASH, sd_hash, return_json=False)
         d.addCallback(self._reflect)
-        d.addCallbacks(lambda _: self._render_response(True, OK_CODE), lambda err: self._render_response(err.getTraceback(), OK_CODE))
+        d.addCallbacks(
+            lambda _: self._render_response(True, OK_CODE),
+            lambda err: self._render_response(err.getTraceback(), OK_CODE))
         return d
 
     def jsonrpc_get_blob_hashes(self):
@@ -2343,8 +2336,9 @@ class Daemon(AuthJSONRPCServer):
         d = self._resolve_name(name, force_refresh=True)
         d.addCallback(get_sd_hash)
         d.addCallback(self._download_sd_blob)
-        d.addCallbacks(lambda descriptor: [blob.get('blob_hash') for blob in descriptor['blobs']],
+        d.addCallbacks(
-                       lambda _: [])
+            lambda descriptor: [blob.get('blob_hash') for blob in descriptor['blobs']],
+            lambda _: [])
         d.addCallback(self.session.blob_tracker.get_availability_for_blobs)
         d.addCallback(_get_mean)
         d.addCallback(lambda result: self._render_response(result, OK_CODE))
@@ -2541,7 +2535,7 @@ class _ResolveNameHelper(object):
         if self.need_fresh_stream():
             log.info("Resolving stream info for lbry://%s", self.name)
             d = self.wallet.get_stream_info_for_name(self.name)
-            d.addCallbacks(self._cache_stream_info, lambda _: defer.fail(UnknownNameError))
+            d.addCallback(self._cache_stream_info)
         else:
             log.debug("Returning cached stream info for lbry://%s", self.name)
             d = defer.succeed(self.name_data['claim_metadata'])
@@ -2709,3 +2703,15 @@ def get_lbry_file_search_value(p):
         if value:
             return searchtype, value
     raise NoValidSearch()
+
+
+def handle_failure(err, msg):
+    log_support.failure(err, log, msg)
+    # TODO: Is this a module? It looks like it:
+    #
+    # In [1]: import twisted.web.server
+    # In [2]: twisted.web.server.failure
+    # Out[2]: <module 'twisted.python.failure' from '.../site-packages/twisted/python/failure.pyc'>
+    #
+    # If so, maybe we should return something else.
+    return server.failure

lbrynet/lbrynet_daemon/DaemonControl.py

@@ -5,32 +5,23 @@ import webbrowser
 import sys
 
 from twisted.web import server, guard
-from twisted.internet import defer, reactor
+from twisted.internet import defer, reactor, error
 from twisted.cred import portal
-
 from jsonrpc.proxy import JSONRPCProxy
 
 from lbrynet.lbrynet_daemon.auth.auth import PasswordChecker, HttpPasswordRealm
 from lbrynet.lbrynet_daemon.auth.util import initialize_api_key_file
+from lbrynet import conf
 from lbrynet.core import log_support
 from lbrynet.core import utils
 from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer
 from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest
 from lbrynet.conf import settings
 
-log_dir = settings.data_dir
 
-if not os.path.isdir(log_dir):
-    os.mkdir(log_dir)
-
-lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
-if getattr(sys, 'frozen', False) and os.name == "nt":
-    os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(os.path.dirname(sys.executable), "cacert.pem")
-
-
 def test_internet_connection():
     return utils.check_connection()
 
@@ -56,45 +47,30 @@ def start():
                         help="lbrycrd or lbryum, default lbryum",
                         type=str,
                         default='lbryum')
-
+    parser.add_argument("--ui", help="path to custom UI folder", default=None)
-    parser.add_argument("--ui",
+    parser.add_argument(
-                        help="path to custom UI folder",
+        "--branch",
-                        default=None)
+        help='Branch of lbry-web-ui repo to use, defaults to {}'.format(settings.ui_branch),
-
+        default=settings.ui_branch)
-    parser.add_argument("--branch",
+    parser.add_argument('--no-launch', dest='launchui', action="store_false")
-                        help="Branch of lbry-web-ui repo to use, defaults on master",
+    parser.add_argument("--http-auth", dest="useauth", action="store_true")
-                        default=settings.ui_branch)
+    parser.add_argument(
-
+        '--log-to-console', dest='logtoconsole', action='store_true',
-    parser.add_argument("--http-auth",
+        help=('Set to enable console logging. Set the --verbose flag '
-                        dest="useauth",
+              ' to enable more detailed console logging'))
-                        action="store_true")
+    parser.add_argument(
-
+        '--quiet', dest='quiet', action="store_true",
-    parser.add_argument('--no-launch',
+        help=('If log-to-console is not set, setting this disables all console output. '
-                        dest='launchui',
+              'If log-to-console is set, this argument is ignored'))
-                        action="store_false")
+    parser.add_argument(
-
+        '--verbose', nargs="*",
-    parser.add_argument('--log-to-console',
+        help=('Enable debug output. Optionally specify loggers for which debug output '
-                        dest='logtoconsole',
+              'should selectively be applied.'))
-                        action="store_true")
-
-    parser.add_argument('--quiet',
-                        dest='quiet',
-                        action="store_true")
-
-    parser.add_argument('--verbose',
-                        action='store_true',
-                        help='enable more debug output for the console')
-
-    parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False, useauth=settings.use_auth_http)
     args = parser.parse_args()
 
-    log_support.configure_file_handler(lbrynet_log)
+    utils.setup_certs_for_windows()
-    log_support.configure_loggly_handler()
+    lbrynet_log = conf.get_log_filename()
-    if args.logtoconsole:
+    log_support.configure_logging(lbrynet_log, args.logtoconsole, args.verbose)
-        log_support.configure_console(level='DEBUG')
-    log_support.disable_third_party_loggers()
-    if not args.verbose:
-        log_support.disable_noisy_loggers()
 
     to_pass = {}
     settings_path = os.path.join(settings.data_dir, "daemon_settings.yml")
@@ -106,9 +82,10 @@ def start():
     if args.branch:
         to_pass.update({'ui_branch': args.branch})
     to_pass.update({'use_auth_http': args.useauth})
-    to_pass.update({'wallet': args.wallet})
+    to_pass.update({'wallet_type': args.wallet})
-    print to_pass
+    log.debug('Settings overrides: %s', to_pass)
     settings.update(to_pass)
+    log.debug('Final Settings: %s', settings.__dict__)
 
     try:
         JSONRPCProxy.from_url(settings.API_CONNECTION_STRING).is_running()
@@ -131,29 +108,7 @@ def start():
         print "To quit press ctrl-c or call 'stop' via the API"
 
     if test_internet_connection():
-        lbry = DaemonServer()
+        start_server_and_listen(args.launchui, args.useauth)
-
-        d = lbry.start()
-        if args.launchui:
-            d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS))
-        d.addErrback(log_and_kill)
-
-        if settings.use_auth_http:
-            log.info("Using authenticated API")
-            pw_path = os.path.join(settings.data_dir, ".api_keys")
-            initialize_api_key_file(pw_path)
-            checker = PasswordChecker.load_file(pw_path)
-            realm = HttpPasswordRealm(lbry.root)
-            portal_to_realm = portal.Portal(realm, [checker, ])
-            factory = guard.BasicCredentialFactory('Login to lbrynet api')
-            _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
-        else:
-            log.info("Using non-authenticated API")
-            _lbrynet_server = server.Site(lbry.root)
-
-        lbrynet_server = server.Site(_lbrynet_server)
-        lbrynet_server.requestFactory = DaemonRequest
-        reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE)
         reactor.run()
 
         if not args.logtoconsole and not args.quiet:
@@ -170,5 +125,50 @@ def log_and_kill(failure):
     reactor.stop()
 
 
+def start_server_and_listen(launchui, use_auth, **kwargs):
+    """The primary entry point for launching the daemon.
+
+    Args:
+        launchui: set to true to open a browser window
+        use_auth: set to true to enable http authentication
+        kwargs: passed along to `DaemonServer().start()`
+    """
+    lbry = DaemonServer()
+
+    d = lbry.start(**kwargs)
+    if launchui:
+        d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS))
+    d.addErrback(log_and_kill)
+
+    site_base = get_site_base(use_auth, lbry.root)
+    lbrynet_server = server.Site(site_base)
+    lbrynet_server.requestFactory = DaemonRequest
+    try:
+        reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE)
+    except error.CannotListenError:
+        log.info('Daemon already running, exiting app')
+        sys.exit(1)
+
+
+def get_site_base(use_auth, root):
+    if use_auth:
+        log.info("Using authenticated API")
+        return create_auth_session(root)
+    else:
+        log.info("Using non-authenticated API")
+        return server.Site(root)
+
+
+def create_auth_session(root):
+    pw_path = os.path.join(settings.data_dir, ".api_keys")
+    initialize_api_key_file(pw_path)
+    checker = PasswordChecker.load_file(pw_path)
+    realm = HttpPasswordRealm(root)
+    portal_to_realm = portal.Portal(realm, [checker, ])
+    factory = guard.BasicCredentialFactory('Login to lbrynet api')
+    _lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
+    return _lbrynet_server
+
+
 if __name__ == "__main__":
     start()

lbrynet/lbrynet_daemon/DaemonServer.py

@@ -1,29 +1,21 @@
 import logging
 import os
-import sys
 
-from appdirs import user_data_dir
 from twisted.internet import defer
+
+from lbrynet import conf
 from lbrynet.lbrynet_daemon.Daemon import Daemon
 from lbrynet.lbrynet_daemon.Resources import LBRYindex, HostedEncryptedFile, EncryptedFileUpload
 from lbrynet.conf import settings
 
 
-# TODO: omg, this code is essentially duplicated in Daemon
-if sys.platform != "darwin":
-    data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-else:
-    data_dir = user_data_dir("LBRY")
-if not os.path.isdir(data_dir):
-    os.mkdir(data_dir)
-
-lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
 class DaemonServer(object):
     def _setup_server(self):
-        self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active"))
+        ui_path = os.path.join(conf.get_data_dir(), "lbry-ui", "active")
+        self.root = LBRYindex(ui_path)
         self._api = Daemon(self.root)
         self.root.putChild("view", HostedEncryptedFile(self._api))
         self.root.putChild("upload", EncryptedFileUpload(self._api))

lbrynet/lbrynet_daemon/Downloader.py

@@ -1,10 +1,8 @@
 import json
 import logging
 import os
-import sys
 
 from copy import deepcopy
-from appdirs import user_data_dir
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
 
@@ -20,28 +18,24 @@ DOWNLOAD_TIMEOUT_CODE = 'timeout'
 DOWNLOAD_RUNNING_CODE = 'running'
 DOWNLOAD_STOPPED_CODE = 'stopped'
 STREAM_STAGES = [
-                    (INITIALIZING_CODE, 'Initializing...'),
+    (INITIALIZING_CODE, 'Initializing...'),
-                    (DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
+    (DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
-                    (DOWNLOAD_RUNNING_CODE, 'Started stream'),
+    (DOWNLOAD_RUNNING_CODE, 'Started stream'),
-                    (DOWNLOAD_STOPPED_CODE, 'Paused stream'),
+    (DOWNLOAD_STOPPED_CODE, 'Paused stream'),
-                    (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
+    (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
-                ]
+]
 
-if sys.platform != "darwin":
-    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-else:
-    log_dir = user_data_dir("LBRY")
 
-if not os.path.isdir(log_dir):
-    os.mkdir(log_dir)
-
-lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 
 class GetStream(object):
-    def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager,
+    def __init__(self, sd_identifier, session, wallet,
-                 max_key_fee, data_rate=0.5, timeout=settings.download_timeout, download_directory=None, file_name=None):
+                 lbry_file_manager, exchange_rate_manager,
+                 max_key_fee, data_rate=0.5, timeout=None,
+                 download_directory=None, file_name=None):
+        if timeout is None:
+            timeout = settings.download_timeout
         self.wallet = wallet
         self.resolved_name = None
         self.description = None

lbrynet/lbrynet_daemon/Publisher.py

@@ -1,10 +1,9 @@
 import logging
 import mimetypes
 import os
-import sys
 import random
 
-from appdirs import user_data_dir
+from twisted.internet import threads, defer, reactor
 
 from lbrynet.core.Error import InsufficientFundsError
 from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file
@@ -13,17 +12,8 @@ from lbrynet.metadata.Metadata import Metadata
 from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
 from lbrynet import reflector
 from lbrynet.conf import settings
-from twisted.internet import threads, defer, reactor
 
-if sys.platform != "darwin":
-    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-else:
-    log_dir = user_data_dir("LBRY")
 
-if not os.path.isdir(log_dir):
-    os.mkdir(log_dir)
-
-lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 

lbrynet/lbrynet_daemon/UIManager.py

@@ -15,15 +15,7 @@ from lbryum.version import LBRYUM_VERSION as lbryum_version
 from zipfile import ZipFile
 from appdirs import user_data_dir
 
-if sys.platform != "darwin":
-    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-else:
-    log_dir = user_data_dir("LBRY")
 
-if not os.path.isdir(log_dir):
-    os.mkdir(log_dir)
-
-lbrynet_log = os.path.join(log_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
 

packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py

@@ -55,7 +55,6 @@ class LBRYDaemonApp(AppKit.NSApplication):
         self.statusitem.setMenu_(self.menubarMenu)
         self.statusitem.setToolTip_(settings.APP_NAME)
 
-
         if test_internet_connection():
             if platform.mac_ver()[0] >= "10.10":
                 LBRYNotify("Starting LBRY")
@@ -64,12 +63,7 @@ class LBRYDaemonApp(AppKit.NSApplication):
                 LBRYNotify("LBRY needs an internet connection to start, try again when one is available")
             sys.exit(0)
 
-        lbry = DaemonServer()
+        DaemonControl.start_server_and_listen(launchui=True, use_auth=False)
-        d = lbry.start(use_authentication=False)
-        d.addCallback(lambda _: webbrowser.open(settings.UI_ADDRESS))
-        lbrynet_server = server.Site(lbry.root)
-        lbrynet_server.requestFactory = DaemonRequest
-        reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE)
 
     def openui_(self, sender):
         webbrowser.open(settings.UI_ADDRESS)

packaging/osx/lbry-osx-app/lbrygui/main.py

@@ -4,32 +4,22 @@ install(runner=AppHelper.runEventLoop)
 from twisted.internet import reactor
 
 import logging
-import sys
-import os
-from appdirs import user_data_dir
 
+from lbrynet import conf
+from lbrynet.core import log_support
 from LBRYApp import LBRYDaemonApp
 
-if sys.platform != "darwin":
-    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-else:
-    log_dir = user_data_dir("LBRY")
 
-if not os.path.isdir(log_dir):
+log = logging.getLogger()
-    os.mkdir(log_dir)
-
-LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
-
-log = logging.getLogger(__name__)
-handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, backupCount=5)
-log.addHandler(handler)
-logging.basicConfig(level=logging.INFO)
 
 
 def main():
+    log_file = conf.get_log_filename()
+    log_support.configure_logging(log_file, console=False)
     app = LBRYDaemonApp.sharedApplication()
     reactor.addSystemEventTrigger("after", "shutdown", AppHelper.stopEventLoop)
     reactor.run()
 
+
 if __name__ == "__main__":
     main()

packaging/windows/lbry-win32-app/LBRYWin32App.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import socket
 import sys
 import threading
 import webbrowser
@@ -11,31 +10,24 @@ import win32gui_struct
 from jsonrpc.proxy import JSONRPCProxy
 from twisted.internet import reactor, error
 from twisted.web import server
-import twisted
 
 try:
     import winxpgui as win32gui
 except ImportError:
     import win32gui
 
+from lbrynet import conf
+from lbrynet.core import log_support
 from lbrynet.core import utils
+from lbrynet.lbrynet_daemon import DaemonControl
 from lbrynet.lbrynet_daemon.DaemonServer import DaemonServer
 from lbrynet.lbrynet_daemon.DaemonRequest import DaemonRequest
 from lbrynet.conf import settings
 from packaging.uri_handler.LBRYURIHandler import LBRYURIHandler
 
 
-# TODO: omg, this code is essentially duplicated in LBRYDaemon
-data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
-if not os.path.isdir(data_dir):
-    os.mkdir(data_dir)
-
-lbrynet_log = os.path.join(data_dir, settings.LOG_FILE_NAME)
 log = logging.getLogger(__name__)
 
-if getattr(sys, 'frozen', False) and os.name == "nt":
-    os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(os.path.dirname(sys.executable), "cacert.pem")
-
 
 def test_internet_connection():
     return utils.check_connection()
@@ -278,19 +270,14 @@ def main(lbry_name=None):
     systray_thread.daemon = True
     systray_thread.start()
 
-    lbry = DaemonServer()
+    DaemonControl.start_server_and_listen(launchui=True, use_auth=False)
-    d = lbry.start(use_authentication=False)
-    d.addCallback(lambda _: LBRYURIHandler.open_address(lbry_name))
-    lbrynet_server = server.Site(lbry.root)
-    lbrynet_server.requestFactory = DaemonRequest
-    try:
-        reactor.listenTCP(settings.api_port, lbrynet_server, interface=settings.API_INTERFACE)
-    except error.CannotListenError:
-        log.info('Daemon already running, exiting app')
-        sys.exit(1)
     reactor.run()
 
+
 if __name__ == '__main__':
+    utils.setup_certs_for_windows()
+    log_file = conf.get_log_filename()
+    log_support.configure_logging(log_file, console=False)
     lbry_daemon = JSONRPCProxy.from_url(settings.API_CONNECTION_STRING)
 
     try: