refactor default directory setup, remove lbrynet.androidhelpers

fixes https://github.com/lbryio/lbry/issues/1606

lbrynet/androidhelpers/__init__.py

@@ -1 +0,0 @@
-from . import paths

lbrynet/androidhelpers/paths.py

@@ -1,26 +0,0 @@
-# App root files dir
-# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
-def android_files_dir():
-    return None
-
-
-# Base internal storage path
-def android_internal_storage_dir():
-    return None
-
-
-# Base external (SD card, if present) storage path
-def android_external_storage_dir():
-    return None
-
-
-# Internal device storage (private app folder)
-# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
-def android_app_internal_storage_dir():
-    return None
-
-
-# External (app folder on SD card, if present) storage
-# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
-def android_app_external_storage_dir():
-    return None

lbrynet/conf.py

@@ -1,24 +1,15 @@
-import base58
-import json
-import logging
 import os
 import re
 import sys
-import yaml
+import typing
+import json
+import logging
 import envparse
+import base58
+import yaml
 from appdirs import user_data_dir, user_config_dir
 from lbrynet import utils
 from lbrynet.p2p.Error import InvalidCurrencyError, NoSuchDirectoryError
-from lbrynet.androidhelpers.paths import (
-    android_internal_storage_dir,
-    android_app_internal_storage_dir
-)
-
-try:
-    from lbrynet.winpaths import get_path, FOLDERID, UserHandle
-except (ImportError, ValueError, NameError):
-    # Android platform: NameError: name 'c_wchar' is not defined
-    pass
 
 log = logging.getLogger(__name__)
 
@@ -59,89 +50,68 @@ settings_encoders = {
     '.yml': yaml.safe_dump
 }
 
-# set by CLI when the user specifies an alternate config file path
-conf_file = None
-
-
-def _get_old_directories(platform_type):
-    directories = {}
-    if platform_type == WINDOWS:
-        appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
-        directories['data'] = os.path.join(appdata, 'lbrynet')
-        directories['lbryum'] = os.path.join(appdata, 'lbryum')
-        directories['download'] = get_path(FOLDERID.Downloads, UserHandle.current)
-    elif platform_type == DARWIN:
-        directories['data'] = user_data_dir('LBRY')
-        directories['lbryum'] = os.path.expanduser('~/.lbryum')
-        directories['download'] = os.path.expanduser('~/Downloads')
-    elif platform_type == LINUX:
-        directories['data'] = os.path.expanduser('~/.lbrynet')
-        directories['lbryum'] = os.path.expanduser('~/.lbryum')
-        directories['download'] = os.path.expanduser('~/Downloads')
-    else:
-        raise ValueError('unknown platform value')
-    return directories
-
-
-def _get_new_directories(platform_type):
-    directories = {}
-    if platform_type == ANDROID:
-        directories['data'] = '%s/lbrynet' % android_app_internal_storage_dir()
-        directories['lbryum'] = '%s/lbryum' % android_app_internal_storage_dir()
-        directories['download'] = '%s/Download' % android_internal_storage_dir()
-    elif platform_type == WINDOWS:
-        directories['data'] = user_data_dir('lbrynet', 'lbry')
-        directories['lbryum'] = user_data_dir('lbryum', 'lbry')
-        directories['download'] = get_path(FOLDERID.Downloads, UserHandle.current)
-    elif platform_type == DARWIN:
-        directories = _get_old_directories(platform_type)
-    elif platform_type == LINUX:
-        directories['data'] = user_data_dir('lbry/lbrynet')
-        directories['lbryum'] = user_data_dir('lbry/lbryum')
-        try:
-            with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
-                down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
-                down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir)
-                directories['download'] = re.sub('\"', '', down_dir)
-        except EnvironmentError:
-            directories['download'] = os.getenv('XDG_DOWNLOAD_DIR')
-
-        if not directories['download']:
-            directories['download'] = os.path.expanduser('~/Downloads')
-    else:
-        raise ValueError('unknown platform value')
-    return directories
-
-
 if 'ANDROID_ARGUMENT' in os.environ:
     # https://github.com/kivy/kivy/blob/master/kivy/utils.py#L417-L421
     platform = ANDROID
-    dirs = _get_new_directories(ANDROID)
 elif 'darwin' in sys.platform:
     platform = DARWIN
-    dirs = _get_old_directories(DARWIN)
 elif 'win' in sys.platform:
     platform = WINDOWS
-    if os.path.isdir(_get_old_directories(WINDOWS)['data']) or \
-       os.path.isdir(_get_old_directories(WINDOWS)['lbryum']):
-        dirs = _get_old_directories(WINDOWS)
-    else:
-        dirs = _get_new_directories(WINDOWS)
 else:
     platform = LINUX
-    if os.path.isdir(_get_old_directories(LINUX)['data']) or \
-       os.path.isdir(_get_old_directories(LINUX)['lbryum']):
-        dirs = _get_old_directories(LINUX)
-    else:
-        dirs = _get_new_directories(LINUX)
-
-default_data_dir = dirs['data']
-default_lbryum_dir = dirs['lbryum']
-default_download_dir = dirs['download']
 
 ICON_PATH = 'icons' if platform is WINDOWS else 'app.icns'
 
 
+def get_windows_directories() -> typing.Tuple[str, str, str]:
+    from lbrynet.winpaths import get_path, FOLDERID, UserHandle
+
+    download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
+
+    # old
+    appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
+    data_dir = os.path.join(appdata, 'lbrynet')
+    lbryum_dir = os.path.join(appdata, 'lbryum')
+    if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
+        return data_dir, lbryum_dir, download_dir
+
+    # new
+    data_dir = user_data_dir('lbrynet', 'lbry')
+    lbryum_dir = user_data_dir('lbryum', 'lbry')
+    download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
+    return data_dir, lbryum_dir, download_dir
+
+
+def get_darwin_directories() -> typing.Tuple[str, str, str]:
+    data_dir = user_data_dir('LBRY')
+    lbryum_dir = os.path.expanduser('~/.lbryum')
+    download_dir = os.path.expanduser('~/Downloads')
+    return data_dir, lbryum_dir, download_dir
+
+
+def get_linux_directories() -> typing.Tuple[str, str, str]:
+    download_dir = None
+    try:
+        with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
+            down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
+            down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir)
+            download_dir = re.sub('\"', '', down_dir)
+    except EnvironmentError:
+        download_dir = os.getenv('XDG_DOWNLOAD_DIR')
+
+    if not download_dir:
+        download_dir = os.path.expanduser('~/Downloads')
+
+    # old
+    data_dir = os.path.expanduser('~/.lbrynet')
+    lbryum_dir = os.path.expanduser('~/.lbryum')
+    if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
+        return data_dir, lbryum_dir, download_dir
+
+    # new
+    return user_data_dir('lbry/lbrynet'), user_data_dir('lbry/lbryum'), download_dir
+
+
 def server_port(server_and_port):
     server, port = server_and_port.split(':')
     return server, int(port)
@@ -244,18 +214,15 @@ ADJUSTABLE_SETTINGS = {
     # will not be made automatically)
     'auto_renew_claim_height_delta': (int, 0),
     'cache_time': (int, 150),
-    'data_dir': (str, default_data_dir),
     'data_rate': (float, .0001),  # points/megabyte
     'delete_blobs_on_remove': (bool, True),
     'dht_node_port': (int, 4444),
-    'download_directory': (str, default_download_dir),
     'download_timeout': (int, 180),
     'download_mirrors': (list, ['blobs.lbry.io']),
     'is_generous_host': (bool, True),
     'announce_head_blobs_only': (bool, True),
     'concurrent_announcers': (int, DEFAULT_CONCURRENT_ANNOUNCERS),
     'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list, server_list_reverse),
-    'lbryum_wallet_dir': (str, default_lbryum_dir),
     'max_connections_per_stream': (int, 5),
     'seek_head_blob_first': (bool, True),
     # TODO: writing json on the cmd line is a pain, come up with a nicer
@@ -290,16 +257,36 @@ ADJUSTABLE_SETTINGS = {
 }
 
 
-class Config:
+optional_str = typing.Optional[str]
-    def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None,
-                 environment=None, cli_settings=None):
 
+class Config:
+    def __init__(self, fixed_defaults, adjustable_defaults: typing.Dict, persisted_settings=None, environment=None,
+                 cli_settings=None, data_dir: optional_str = None, wallet_dir: optional_str = None,
+                 download_dir: optional_str = None, file_name: optional_str = None):
         self._installation_id = None
         self._session_id = base58.b58encode(utils.generate_id()).decode()
         self._node_id = None
 
         self._fixed_defaults = fixed_defaults
-        self._adjustable_defaults = adjustable_defaults
+
+        # copy the default adjustable settings
+        self._adjustable_defaults = {k: v for k, v in adjustable_defaults.items()}
+
+        default_data_dir, default_wallet_dir, default_download_dir = None, None, None
+        # set the os specific default directories
+        if platform is WINDOWS:
+            default_data_dir, default_wallet_dir, default_download_dir = get_windows_directories()
+        elif platform is DARWIN:
+            default_data_dir, default_wallet_dir, default_download_dir = get_darwin_directories()
+        elif platform is LINUX:
+            default_data_dir, default_wallet_dir, default_download_dir = get_linux_directories()
+        else:
+            assert None not in [data_dir, wallet_dir, download_dir]
+
+        self.data_dir = data_dir or default_data_dir
+        self.download_dir = download_dir or default_download_dir
+        self.wallet_dir = wallet_dir or default_wallet_dir
+        self.file_name = file_name or self.get_valid_settings_filename()
 
         self._data = {
             TYPE_DEFAULT: {},  # defaults
@@ -483,16 +470,15 @@ class Config:
             }
 
     def save_conf_file_settings(self):
-        path = conf_file or self.get_valid_settings_filename()
         # reverse the conversions done after loading the settings from the conf
         # file
         rev = self._convert_conf_file_lists_reverse(self._data[TYPE_PERSISTED])
-        ext = os.path.splitext(path)[1]
+        ext = os.path.splitext(self.file_name)[1]
         encoder = settings_encoders.get(ext, False)
         if not encoder:
             raise ValueError('Unknown settings format: {}. Available formats: {}'
                              .format(ext, list(settings_encoders.keys())))
-        with open(path, 'w') as settings_file:
+        with open(os.path.join(self.data_dir, self.file_name), 'w') as settings_file:
             settings_file.write(encoder(rev))
 
     @staticmethod
@@ -520,14 +506,13 @@ class Config:
         settings.node_id = settings.get_node_id()
 
     def load_conf_file_settings(self):
-        path = conf_file or self.get_valid_settings_filename()
+        self._read_conf_file(os.path.join(self.data_dir, self.file_name))
-        self._read_conf_file(path)
         # initialize members depending on config file
         self.initialize_post_conf_load()
 
     def _read_conf_file(self, path):
         if not path or not os.path.exists(path):
-            return
+            raise FileNotFoundError(path)
         ext = os.path.splitext(path)[1]
         decoder = settings_decoders.get(ext, False)
         if not decoder:
@@ -562,13 +547,15 @@ class Config:
         # although there is a risk of a race condition here we don't
         # expect there to be multiple processes accessing this
         # directory so the risk can be ignored
-        if not os.path.isdir(self['data_dir']):
+        if not os.path.isdir(self.data_dir):
-            os.makedirs(self['data_dir'])
+            os.makedirs(self.data_dir)
-        return self['data_dir']
+        if not os.path.isdir(os.path.join(self.data_dir, "blobfiles")):
+            os.makedirs(os.path.join(self.data_dir, "blobfiles"))
+        return self.data_dir
 
     def ensure_wallet_dir(self):
-        if not os.path.isdir(self['lbryum_wallet_dir']):
+        if not os.path.isdir(self.wallet_dir):
-            os.makedirs(self['lbryum_wallet_dir'])
+            os.makedirs(self.wallet_dir)
 
     def get_log_filename(self):
         """
@@ -624,7 +611,7 @@ class Config:
         return self._session_id
 
 
-settings = None  # type: Config
+settings: Config = None
 
 
 def get_default_env():
@@ -639,10 +626,15 @@ def get_default_env():
     return Env(**env_defaults)
 
 
-def initialize_settings(load_conf_file=True):
+def initialize_settings(load_conf_file: typing.Optional[bool] = True,
+                        data_dir: optional_str = None, wallet_dir: optional_str = None,
+                        download_dir: optional_str = None):
     global settings
     if settings is None:
         settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS,
-                          environment=get_default_env())
+                          environment=get_default_env(), data_dir=data_dir, wallet_dir=wallet_dir,
+                          download_dir=download_dir)
         if load_conf_file:
             settings.load_conf_file_settings()
+        settings.ensure_data_dir()
+        settings.ensure_wallet_dir()

lbrynet/extras/cli.py

@@ -1,4 +1,16 @@
 import sys
+import os
+import json
+import asyncio
+import argparse
+import typing
+
+# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
+# This needs to happen before anything else
+if 'win' in sys.platform:
+    import certifi
+    os.environ['SSL_CERT_FILE'] = certifi.where()
+
 from twisted.internet import asyncioreactor
 if 'twisted.internet.reactor' not in sys.modules:
     asyncioreactor.install()
@@ -10,28 +22,93 @@ else:
         #    https://github.com/kivy/kivy/issues/4182
         del sys.modules['twisted.internet.reactor']
         asyncioreactor.install()
-        from twisted.internet import reactor
+from twisted.internet import reactor
-
+import logging
-import json
-import asyncio
 from aiohttp.client_exceptions import ClientConnectorError
 from requests.exceptions import ConnectionError
 from docopt import docopt
 from textwrap import dedent
 
-from lbrynet import __name__ as lbrynet_name
+from lbrynet import conf, log_support, __name__ as lbrynet_name
+from lbrynet.utils import check_connection, json_dumps_pretty
 from lbrynet.extras.daemon.Daemon import Daemon
-from lbrynet.extras.daemon.DaemonControl import start as daemon_main
 from lbrynet.extras.daemon.DaemonConsole import main as daemon_console
 from lbrynet.extras.daemon.auth.client import LBRYAPIClient
 from lbrynet.extras.system_info import get_platform
 
+log = logging.getLogger(lbrynet_name)
 
-async def execute_command(method, params, conf_path=None):
+optional_path_getter_type = typing.Optional[typing.Callable[[], str]]
+
+
+def start_daemon(settings: typing.Optional[typing.Dict] = None,
+                 console_output: typing.Optional[bool] = True, verbose: typing.Optional[typing.List[str]] = None,
+                 data_dir: typing.Optional[str] = None, wallet_dir: typing.Optional[str] = None,
+                 download_dir: typing.Optional[str] = None):
+
+    settings = settings or {}
+    conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
+    for k, v in settings.items():
+        conf.settings.update({k, v}, data_types=(conf.TYPE_CLI,))
+
+    log_support.configure_logging(conf.settings.get_log_filename(), console_output, verbose)
+    log_support.configure_loggly_handler()
+    log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
+    log.info("Starting lbrynet-daemon from command line")
+
+    if check_connection():
+        daemon = Daemon()
+        daemon.start_listening()
+        reactor.run()
+    else:
+        log.info("Not connected to internet, unable to start")
+
+
+def start_daemon_with_cli_args(argv=None, data_dir: typing.Optional[str] = None,
+                               wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--http-auth", dest="useauth", action="store_true", default=False
+    )
+    parser.add_argument(
+        '--quiet', dest='quiet', action="store_true",
+        help='Disable all console output.'
+    )
+    parser.add_argument(
+        '--verbose', nargs="*",
+        help=('Enable debug output. Optionally specify loggers for which debug output '
+              'should selectively be applied.')
+    )
+    parser.add_argument(
+        '--version', action="store_true",
+        help='Show daemon version and quit'
+    )
+
+    args = parser.parse_args(argv)
+    settings = {}
+    if args.useauth:
+        settings['use_auth_http'] = True
+
+    verbose = None
+    if args.verbose:
+        verbose = args.verbose
+
+    console_output = not args.quiet
+
+    if args.version:
+        print(json_dumps_pretty(get_platform()))
+        return
+
+    return start_daemon(settings, console_output, verbose, data_dir, wallet_dir, download_dir)
+
+
+async def execute_command(method, params, data_dir: typing.Optional[str] = None,
+                          wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
     # this check if the daemon is running or not
+    conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
     api = None
     try:
-        api = await LBRYAPIClient.get_client(conf_path)
+        api = await LBRYAPIClient.get_client()
         await api.status()
     except (ClientConnectorError, ConnectionError):
         if api:
@@ -119,14 +196,31 @@ def main(argv=None):
         print_help()
         return 1
 
-    conf_path = None
+    data_dir = None
-    if len(argv) and argv[0] == "--conf":
+    if len(argv) and argv[0] == "--data_dir":
         if len(argv) < 2:
-            print("No config file specified for --conf option")
+            print("No directory specified for --data_dir option")
             print_help()
             return 1
+        data_dir = argv[1]
+        argv = argv[2:]
 
-        conf_path = argv[1]
+    wallet_dir = None
+    if len(argv) and argv[0] == "--wallet_dir":
+        if len(argv) < 2:
+            print("No directory specified for --wallet_dir option")
+            print_help()
+            return 1
+        wallet_dir = argv[1]
+        argv = argv[2:]
+
+    download_dir = None
+    if len(argv) and argv[0] == "--download_dir":
+        if len(argv) < 2:
+            print("No directory specified for --data_dir option")
+            print_help()
+            return 1
+        download_dir = argv[1]
         argv = argv[2:]
 
     method, args = argv[0], argv[1:]
@@ -145,7 +239,7 @@ def main(argv=None):
         return 0
 
     elif method == 'start':
-        sys.exit(daemon_main(args, conf_path))
+        sys.exit(start_daemon_with_cli_args(args, data_dir, wallet_dir, download_dir))
 
     elif method == 'console':
         sys.exit(daemon_console())
@@ -167,7 +261,7 @@ def main(argv=None):
     parsed = docopt(fn.__doc__, args)
     params = set_kwargs(parsed)
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(execute_command(method, params, conf_path))
+    loop.run_until_complete(execute_command(method, params, data_dir, wallet_dir, download_dir))
 
     return 0
 

lbrynet/extras/daemon/Components.py

@@ -59,62 +59,18 @@ def from_future(coroutine: asyncio.coroutine) -> defer.Deferred:
     return defer.Deferred.fromFuture(asyncio.ensure_future(coroutine))
 
 
-def get_wallet_config():
+@defer.inlineCallbacks
-    wallet_type = GCS('wallet')
+def get_external_ip():  # used if upnp is disabled or non-functioning
-    if wallet_type == conf.LBRYCRD_WALLET:
+    try:
-        raise ValueError('LBRYcrd Wallet is no longer supported')
+        buf = []
-    elif wallet_type != conf.LBRYUM_WALLET:
+        response = yield treq.get("https://api.lbry.io/ip")
-        raise ValueError(f'Wallet Type {wallet_type} is not valid')
+        yield treq.collect(response, buf.append)
-    lbryum_servers = {address: {'t': str(port)}
+        parsed = json.loads(b"".join(buf).decode())
-                          for address, port in GCS('lbryum_servers')}
+        if parsed['success']:
-    config = {
+            return parsed['data']['ip']
-        'auto_connect': True,
+        return
-        'chain': GCS('blockchain_name'),
+    except Exception as err:
-        'default_servers': lbryum_servers
+        return
-    }
-    if 'use_keyring' in conf.settings:
-        config['use_keyring'] = GCS('use_keyring')
-    if conf.settings['lbryum_wallet_dir']:
-        config['lbryum_path'] = GCS('lbryum_wallet_dir')
-    return config
-
-
-class ConfigSettings:
-    @staticmethod
-    def get_conf_setting(setting_name):
-        return conf.settings[setting_name]
-
-    @staticmethod
-    def get_blobfiles_dir():
-        if conf.settings['BLOBFILES_DIR'] == "blobfiles":
-            return os.path.join(GCS("data_dir"), "blobfiles")
-        else:
-            log.info("Using non-default blobfiles directory: %s", conf.settings['BLOBFILES_DIR'])
-            return conf.settings['BLOBFILES_DIR']
-
-    @staticmethod
-    def get_node_id():
-        return conf.settings.node_id
-
-    @staticmethod
-    @defer.inlineCallbacks
-    def get_external_ip():  # used if upnp is disabled or non-functioning
-        try:
-            buf = []
-            response = yield treq.get("https://api.lbry.io/ip")
-            yield treq.collect(response, buf.append)
-            parsed = json.loads(b"".join(buf).decode())
-            if parsed['success']:
-                return parsed['data']['ip']
-            return
-        except Exception as err:
-            return
-
-
-
-# Shorthand for common ConfigSettings methods
-CS = ConfigSettings
-GCS = ConfigSettings.get_conf_setting
 
 
 class DatabaseComponent(Component):
@@ -146,18 +102,6 @@ class DatabaseComponent(Component):
         # check directories exist, create them if they don't
         log.info("Loading databases")
 
-        if not os.path.exists(GCS('download_directory')):
-            os.mkdir(GCS('download_directory'))
-
-        if not os.path.exists(GCS('data_dir')):
-            os.mkdir(GCS('data_dir'))
-            self._write_db_revision_file(self.get_current_db_revision())
-            log.debug("Created the db revision file: %s", self.get_revision_filename())
-
-        if not os.path.exists(CS.get_blobfiles_dir()):
-            os.mkdir(CS.get_blobfiles_dir())
-            log.debug("Created the blobfile directory: %s", str(CS.get_blobfiles_dir()))
-
         if not os.path.exists(self.get_revision_filename()):
             log.warning("db_revision file not found. Creating it")
             self._write_db_revision_file(self.get_current_db_revision())
@@ -174,13 +118,13 @@ class DatabaseComponent(Component):
             from lbrynet.extras.daemon.migrator import dbmigrator
             log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
             yield threads.deferToThread(
-                dbmigrator.migrate_db, GCS('data_dir'), old_revision, self.get_current_db_revision()
+                dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision()
             )
             self._write_db_revision_file(self.get_current_db_revision())
             log.info("Finished upgrading the databases.")
 
         # start SQLiteStorage
-        self.storage = SQLiteStorage(GCS('data_dir'))
+        self.storage = SQLiteStorage(conf.settings.data_dir)
         yield self.storage.setup()
 
     @defer.inlineCallbacks
@@ -198,9 +142,9 @@ class HeadersComponent(Component):
 
     def __init__(self, component_manager):
         super().__init__(component_manager)
-        self.headers_dir = os.path.join(conf.settings['lbryum_wallet_dir'], 'lbc_mainnet')
+        self.headers_dir = os.path.join(conf.settings.wallet_dir, 'lbc_mainnet')
         self.headers_file = os.path.join(self.headers_dir, 'headers')
-        self.old_file = os.path.join(conf.settings['lbryum_wallet_dir'], 'blockchain_headers')
+        self.old_file = os.path.join(conf.settings.wallet_dir, 'blockchain_headers')
         self._downloading_headers = None
         self._headers_progress_percent = 0
 
@@ -260,7 +204,7 @@ class HeadersComponent(Component):
         ledger = SimpleNamespace()
         ledger.config = {
             'default_servers': conf.settings['lbryum_servers'],
-            'data_path': conf.settings['lbryum_wallet_dir']
+            'data_path': conf.settings.wallet_dir
         }
         net = Network(ledger)
         first_connection = net.on_connected.first
@@ -395,7 +339,7 @@ class BlobComponent(Component):
             dht_node = self.component_manager.get_component(DHT_COMPONENT)
             if dht_node:
                 datastore = dht_node._dataStore
-        self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, datastore)
+        self.blob_manager = DiskBlobManager(os.path.join(conf.settings.data_dir, "blobfiles"), storage, datastore)
         return self.blob_manager.setup()
 
     def stop(self):
@@ -428,34 +372,34 @@ class DHTComponent(Component):
 
     def get_status(self):
         return {
-            'node_id': binascii.hexlify(CS.get_node_id()),
+            'node_id': binascii.hexlify(conf.settings.get_node_id()),
             'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
         }
 
     @defer.inlineCallbacks
     def start(self):
         self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
-        self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", GCS("peer_port"))
+        self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"])
-        self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", GCS("dht_node_port"))
+        self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"])
-        node_id = CS.get_node_id()
+        node_id = conf.settings.get_node_id()
         if node_id is None:
             node_id = generate_id()
         external_ip = self.upnp_component.external_ip
         if not external_ip:
             log.warning("UPnP component failed to get external ip")
-            external_ip = yield CS.get_external_ip()
+            external_ip = yield get_external_ip()
             if not external_ip:
                 log.warning("failed to get external ip")
 
         self.dht_node = Node(
             node_id=node_id,
-            udpPort=GCS('dht_node_port'),
+            udpPort=conf.settings['dht_node_port'],
             externalUDPPort=self.external_udp_port,
             externalIP=external_ip,
             peerPort=self.external_peer_port
         )
 
-        yield self.dht_node.start(GCS('known_dht_nodes'), block_on_join=False)
+        yield self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False)
         log.info("Started the dht")
 
     @defer.inlineCallbacks
@@ -565,7 +509,7 @@ class FileManagerComponent(Component):
             blob_manager,
             storage,
             wallet,
-            GCS('download_directory')
+            conf.settings.download_dir
         )
         yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
 
@@ -598,7 +542,7 @@ class PeerProtocolServerComponent(Component):
     def start(self):
         wallet = self.component_manager.get_component(WALLET_COMPONENT)
         upnp = self.component_manager.get_component(UPNP_COMPONENT)
-        peer_port = GCS('peer_port')
+        peer_port = conf.settings['peer_port']
         query_handlers = {
             handler.get_primary_query_identifier(): handler for handler in [
                 BlobRequestHandlerFactory(
@@ -640,7 +584,7 @@ class ReflectorComponent(Component):
 
     def __init__(self, component_manager):
         super().__init__(component_manager)
-        self.reflector_server_port = GCS('reflector_port')
+        self.reflector_server_port = conf.settings['reflector_port']
         self.reflector_server = None
 
     @property
@@ -673,9 +617,9 @@ class UPnPComponent(Component):
 
     def __init__(self, component_manager):
         super().__init__(component_manager)
-        self._int_peer_port = GCS('peer_port')
+        self._int_peer_port = conf.settings['peer_port']
-        self._int_dht_node_port = GCS('dht_node_port')
+        self._int_dht_node_port = conf.settings['dht_node_port']
-        self.use_upnp = GCS('use_upnp')
+        self.use_upnp = conf.settings['use_upnp']
         self.upnp = None
         self.upnp_redirects = {}
         self.external_ip = None
@@ -719,7 +663,7 @@ class UPnPComponent(Component):
 
         if external_ip == "0.0.0.0" or not external_ip:
             log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
-            external_ip = yield CS.get_external_ip()
+            external_ip = yield get_external_ip()
         if self.external_ip and self.external_ip != external_ip:
             log.info("external ip changed from %s to %s", self.external_ip, external_ip)
         self.external_ip = external_ip
@@ -775,7 +719,7 @@ class UPnPComponent(Component):
     def start(self):
         log.info("detecting external ip")
         if not self.use_upnp:
-            self.external_ip = yield CS.get_external_ip()
+            self.external_ip = yield get_external_ip()
             return
         success = False
         yield self._maintain_redirects()

lbrynet/extras/daemon/Daemon.py

@@ -166,9 +166,9 @@ def sort_claim_results(claims):
 def is_first_run():
     if os.path.isfile(conf.settings.get_db_revision_filename()):
         return False
-    if os.path.isfile(os.path.join(conf.settings['data_dir'], 'lbrynet.sqlite')):
+    if os.path.isfile(os.path.join(conf.settings.data_dir, 'lbrynet.sqlite')):
         return False
-    if os.path.isfile(os.path.join(conf.settings['lbryum_wallet_dir'], 'blockchain_headers')):
+    if os.path.isfile(os.path.join(conf.settings.wallet_dir, 'blockchain_headers')):
         return False
     return True
 

lbrynet/extras/daemon/DaemonControl.py

@@ -1,72 +0,0 @@
-import os
-import sys
-
-# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
-# This needs to happen before anything else
-if 'win' in sys.platform:
-    import certifi
-    os.environ['SSL_CERT_FILE'] = certifi.where()
-
-
-import argparse
-import logging.handlers
-
-from twisted.internet import reactor
-from lbrynet import utils, conf, log_support
-from lbrynet.extras import system_info
-from lbrynet.extras.daemon.Daemon import Daemon
-
-log = logging.getLogger(__name__)
-
-
-def test_internet_connection():
-    return utils.check_connection()
-
-
-def start(argv=None, conf_path=None):
-    if conf_path is not None:
-        conf.conf_file = conf_path
-
-    conf.initialize_settings()
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
-    )
-    parser.add_argument(
-        '--quiet', dest='quiet', action="store_true",
-        help='Disable all console output.'
-    )
-    parser.add_argument(
-        '--verbose', nargs="*",
-        help=('Enable debug output. Optionally specify loggers for which debug output '
-              'should selectively be applied.')
-    )
-    parser.add_argument(
-        '--version', action="store_true",
-        help='Show daemon version and quit'
-    )
-
-    args = parser.parse_args(argv)
-    if args.useauth:
-        conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,))
-
-    if args.version:
-        version = system_info.get_platform()
-        version['installation_id'] = conf.settings.installation_id
-        print(utils.json_dumps_pretty(version))
-        return
-
-    lbrynet_log = conf.settings.get_log_filename()
-    log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
-    log_support.configure_loggly_handler()
-    log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
-
-    log.info("Starting lbrynet-daemon from command line")
-
-    if test_internet_connection():
-        daemon = Daemon()
-        daemon.start_listening()
-        reactor.run()
-    else:
-        log.info("Not connected to internet, unable to start")

lbrynet/extras/daemon/Downloader.py

@@ -41,7 +41,7 @@ class GetStream:
         self.data_rate = data_rate or conf.settings['data_rate']
         self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
         self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
-        self.download_directory = conf.settings['download_directory']
+        self.download_directory = conf.settings.download_dir
         self.timeout_counter = 0
         self.code = None
         self.sd_hash = None

lbrynet/extras/daemon/auth/client.py

@@ -125,9 +125,6 @@ class AuthAPIClient:
 
 class LBRYAPIClient:
     @staticmethod
-    def get_client(conf_path=None):
+    def get_client():
-        conf.conf_file = conf_path
-        if not conf.settings:
-            conf.initialize_settings()
         return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \
             UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())

lbrynet/extras/daemon/auth/keyring.py

@@ -73,8 +73,8 @@ class Keyring:
 
     @classmethod
     def load_from_disk(cls):
-        api_key_path = os.path.join(conf.settings['data_dir'], 'auth_token')
+        api_key_path = os.path.join(conf.settings.data_dir, 'auth_token')
-        api_ssl_cert_path = os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem')
+        api_ssl_cert_path = os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem')
         if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path):
             return
         with open(api_key_path, 'rb') as f:
@@ -122,10 +122,10 @@ class Keyring:
 
         auth_token = APIKey.create(seed=None, name="api")
 
-        with open(os.path.join(conf.settings['data_dir'], 'auth_token'), 'wb') as f:
+        with open(os.path.join(conf.settings.data_dir, 'auth_token'), 'wb') as f:
             f.write(auth_token.secret.encode())
 
-        with open(os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem'), 'wb') as f:
+        with open(os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem'), 'wb') as f:
             f.write(public_certificate.encode())
 
         return cls(auth_token, public_certificate, private_certificate)

lbrynet/extras/daemon/migrator/migrate5to6.py

@@ -7,8 +7,6 @@ from lbrynet.schema.decode import smart_decode
 
 log = logging.getLogger(__name__)
 
-download_directory = conf.settings['download_directory']
-
 CREATE_TABLES_QUERY = """
             pragma foreign_keys=on;
             pragma journal_mode=WAL;
@@ -190,7 +188,7 @@ def do_migration(db_dir):
         # insert the file
         transaction.execute(
             "insert or ignore into file values (?, ?, ?, ?, ?)",
-            (stream_hash, stream_name, download_directory.encode('hex'),
+            (stream_hash, stream_name, conf.settings.download_dir.encode('hex'),
              data_rate, status)
         )
 

lbrynet/extras/wallet/manager.py

@@ -171,12 +171,12 @@ class LbryWalletManager(BaseWalletManager):
         ledger_config = {
             'auto_connect': True,
             'default_servers': settings['lbryum_servers'],
-            'data_path': settings['lbryum_wallet_dir'],
+            'data_path': settings.wallet_dir,
             'use_keyring': settings['use_keyring'],
             #'db': db
         }
 
-        wallets_directory = os.path.join(settings['lbryum_wallet_dir'], 'wallets')
+        wallets_directory = os.path.join(settings.wallet_dir, 'wallets')
         if not os.path.exists(wallets_directory):
             os.mkdir(wallets_directory)
 

tests/integration/wallet/test_commands.py

@@ -126,10 +126,10 @@ class CommandTestCase(IntegrationTestCase):
         logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
 
         lbry_conf.settings = None
-        lbry_conf.initialize_settings(load_conf_file=False)
+        lbry_conf.initialize_settings(
-        lbry_conf.settings['data_dir'] = self.wallet_node.data_path
+            load_conf_file=False, data_dir=self.wallet_node.data_path, wallet_dir=self.wallet_node.data_path,
-        lbry_conf.settings['lbryum_wallet_dir'] = self.wallet_node.data_path
+            download_dir=self.wallet_node.data_path
-        lbry_conf.settings['download_directory'] = self.wallet_node.data_path
+        )
         lbry_conf.settings['use_upnp'] = False
         lbry_conf.settings['reflect_uploads'] = False
         lbry_conf.settings['blockchain_name'] = 'lbrycrd_regtest'

tests/unit/test_conf.py

@@ -2,8 +2,10 @@ import os
 import json
 import sys
 import tempfile
+import shutil
 from unittest import skipIf
 from twisted.trial import unittest
+from twisted.internet import defer
 from lbrynet import conf
 from lbrynet.p2p.Error import InvalidCurrencyError
 
@@ -15,11 +17,12 @@ class SettingsTest(unittest.TestCase):
     def tearDown(self):
         del os.environ['LBRY_TEST']
 
-    @staticmethod
+    def get_mock_config_instance(self):
-    def get_mock_config_instance():
         settings = {'test': (str, '')}
         env = conf.Env(**settings)
-        return conf.Config({}, settings, environment=env)
+        self.tmp_dir = tempfile.mkdtemp()
+        self.addCleanup(lambda : defer.succeed(shutil.rmtree(self.tmp_dir)))
+        return conf.Config({}, settings, environment=env, data_dir=self.tmp_dir, wallet_dir=self.tmp_dir, download_dir=self.tmp_dir)
 
     def test_envvar_is_read(self):
         settings = self.get_mock_config_instance()
@@ -72,20 +75,19 @@ class SettingsTest(unittest.TestCase):
         out = settings.get('max_key_fee')
         self.assertEqual(out, valid_setting)
 
-
     def test_data_dir(self):
         # check if these directories are returned as string and not unicode
         # otherwise there will be problems when calling os.path.join on
         # unicode directory names with string file names
-        self.assertEqual(str, type(conf.default_download_dir))
+        settings = conf.Config({}, {})
-        self.assertEqual(str, type(conf.default_data_dir))
+        self.assertEqual(str, type(settings.download_dir))
-        self.assertEqual(str, type(conf.default_lbryum_dir))
+        self.assertEqual(str, type(settings.data_dir))
+        self.assertEqual(str, type(settings.wallet_dir))
 
     @skipIf('win' in sys.platform, 'fix me!')
     def test_load_save_config_file(self):
         # setup settings
-        adjustable_settings = {'data_dir': (str, conf.default_data_dir),
+        adjustable_settings = {'lbryum_servers': (list, [])}
-                'lbryum_servers': (list, [])}
         env = conf.Env(**adjustable_settings)
         settings = conf.Config({}, adjustable_settings, environment=env)
         conf.settings = settings
@@ -111,12 +113,14 @@ class SettingsTest(unittest.TestCase):
         settings = self.get_mock_config_instance()
 
         # nonexistent file
-        conf.conf_file = 'monkey.yml'
+        settings.file_name = 'monkey.yml'
         with self.assertRaises(FileNotFoundError):
             settings.load_conf_file_settings()
 
         # invalid extensions
         for filename in ('monkey.yymmll', 'monkey'):
-            conf.conf_file = filename
+            settings.file_name = filename
+            with open(os.path.join(self.tmp_dir, filename), "w"):
+                pass
             with self.assertRaises(ValueError):
                 settings.load_conf_file_settings()