From b8b0d8b1acbaff0c010850534f19834e918fb07a Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Nov 2018 15:56:11 -0500 Subject: [PATCH] refactor default directory setup, remove lbrynet.androidhelpers fixes https://github.com/lbryio/lbry/issues/1606 --- lbrynet/androidhelpers/__init__.py | 1 - lbrynet/androidhelpers/paths.py | 26 --- lbrynet/conf.py | 202 +++++++++--------- lbrynet/extras/cli.py | 122 +++++++++-- lbrynet/extras/daemon/Components.py | 122 +++-------- lbrynet/extras/daemon/Daemon.py | 4 +- lbrynet/extras/daemon/DaemonControl.py | 72 ------- lbrynet/extras/daemon/Downloader.py | 2 +- lbrynet/extras/daemon/auth/client.py | 5 +- lbrynet/extras/daemon/auth/keyring.py | 8 +- lbrynet/extras/daemon/migrator/migrate5to6.py | 4 +- lbrynet/extras/wallet/manager.py | 4 +- tests/integration/wallet/test_commands.py | 8 +- tests/unit/test_conf.py | 26 ++- 14 files changed, 268 insertions(+), 338 deletions(-) delete mode 100644 lbrynet/androidhelpers/__init__.py delete mode 100644 lbrynet/androidhelpers/paths.py delete mode 100644 lbrynet/extras/daemon/DaemonControl.py diff --git a/lbrynet/androidhelpers/__init__.py b/lbrynet/androidhelpers/__init__.py deleted file mode 100644 index abf9a4fcc..000000000 --- a/lbrynet/androidhelpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import paths diff --git a/lbrynet/androidhelpers/paths.py b/lbrynet/androidhelpers/paths.py deleted file mode 100644 index 6b9f9ccc2..000000000 --- a/lbrynet/androidhelpers/paths.py +++ /dev/null @@ -1,26 +0,0 @@ -# App root files dir -# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29 -def android_files_dir(): - return None - - -# Base internal storage path -def android_internal_storage_dir(): - return None - - -# Base external (SD card, if present) storage path -def android_external_storage_dir(): - return None - - -# Internal device storage (private app folder) -# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String) -def android_app_internal_storage_dir(): - return None - - -# External (app folder on SD card, if present) storage -# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String) -def android_app_external_storage_dir(): - return None diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 612cf62ea..52fad2034 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -1,24 +1,15 @@ -import base58 -import json -import logging import os import re import sys -import yaml +import typing +import json +import logging import envparse +import base58 +import yaml from appdirs import user_data_dir, user_config_dir from lbrynet import utils from lbrynet.p2p.Error import InvalidCurrencyError, NoSuchDirectoryError -from lbrynet.androidhelpers.paths import ( - android_internal_storage_dir, - android_app_internal_storage_dir -) - -try: - from lbrynet.winpaths import get_path, FOLDERID, UserHandle -except (ImportError, ValueError, NameError): - # Android platform: NameError: name 'c_wchar' is not defined - pass log = logging.getLogger(__name__) @@ -59,89 +50,68 @@ settings_encoders = { '.yml': yaml.safe_dump } -# set by CLI when the user specifies an alternate config file path -conf_file = None - - -def _get_old_directories(platform_type): - directories = {} - if platform_type == WINDOWS: - appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current) - directories['data'] = os.path.join(appdata, 'lbrynet') - directories['lbryum'] = os.path.join(appdata, 'lbryum') - directories['download'] = get_path(FOLDERID.Downloads, UserHandle.current) - elif platform_type == DARWIN: - directories['data'] = user_data_dir('LBRY') - directories['lbryum'] = os.path.expanduser('~/.lbryum') - directories['download'] = os.path.expanduser('~/Downloads') - elif platform_type == LINUX: - directories['data'] = os.path.expanduser('~/.lbrynet') - directories['lbryum'] = os.path.expanduser('~/.lbryum') - directories['download'] = os.path.expanduser('~/Downloads') - else: - raise ValueError('unknown platform value') - return directories - - -def _get_new_directories(platform_type): - directories = {} - if platform_type == ANDROID: - directories['data'] = '%s/lbrynet' % android_app_internal_storage_dir() - directories['lbryum'] = '%s/lbryum' % android_app_internal_storage_dir() - directories['download'] = '%s/Download' % android_internal_storage_dir() - elif platform_type == WINDOWS: - directories['data'] = user_data_dir('lbrynet', 'lbry') - directories['lbryum'] = user_data_dir('lbryum', 'lbry') - directories['download'] = get_path(FOLDERID.Downloads, UserHandle.current) - elif platform_type == DARWIN: - directories = _get_old_directories(platform_type) - elif platform_type == LINUX: - directories['data'] = user_data_dir('lbry/lbrynet') - directories['lbryum'] = user_data_dir('lbry/lbryum') - try: - with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg: - down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1) - down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir) - directories['download'] = re.sub('\"', '', down_dir) - except EnvironmentError: - directories['download'] = os.getenv('XDG_DOWNLOAD_DIR') - - if not directories['download']: - directories['download'] = os.path.expanduser('~/Downloads') - else: - raise ValueError('unknown platform value') - return directories - - if 'ANDROID_ARGUMENT' in os.environ: # https://github.com/kivy/kivy/blob/master/kivy/utils.py#L417-L421 platform = ANDROID - dirs = _get_new_directories(ANDROID) elif 'darwin' in sys.platform: platform = DARWIN - dirs = _get_old_directories(DARWIN) elif 'win' in sys.platform: platform = WINDOWS - if os.path.isdir(_get_old_directories(WINDOWS)['data']) or \ - os.path.isdir(_get_old_directories(WINDOWS)['lbryum']): - dirs = _get_old_directories(WINDOWS) - else: - dirs = _get_new_directories(WINDOWS) else: platform = LINUX - if os.path.isdir(_get_old_directories(LINUX)['data']) or \ - os.path.isdir(_get_old_directories(LINUX)['lbryum']): - dirs = _get_old_directories(LINUX) - else: - dirs = _get_new_directories(LINUX) - -default_data_dir = dirs['data'] -default_lbryum_dir = dirs['lbryum'] -default_download_dir = dirs['download'] ICON_PATH = 'icons' if platform is WINDOWS else 'app.icns' +def get_windows_directories() -> typing.Tuple[str, str, str]: + from lbrynet.winpaths import get_path, FOLDERID, UserHandle + + download_dir = get_path(FOLDERID.Downloads, UserHandle.current) + + # old + appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current) + data_dir = os.path.join(appdata, 'lbrynet') + lbryum_dir = os.path.join(appdata, 'lbryum') + if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir): + return data_dir, lbryum_dir, download_dir + + # new + data_dir = user_data_dir('lbrynet', 'lbry') + lbryum_dir = user_data_dir('lbryum', 'lbry') + download_dir = get_path(FOLDERID.Downloads, UserHandle.current) + return data_dir, lbryum_dir, download_dir + + +def get_darwin_directories() -> typing.Tuple[str, str, str]: + data_dir = user_data_dir('LBRY') + lbryum_dir = os.path.expanduser('~/.lbryum') + download_dir = os.path.expanduser('~/Downloads') + return data_dir, lbryum_dir, download_dir + + +def get_linux_directories() -> typing.Tuple[str, str, str]: + download_dir = None + try: + with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg: + down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1) + down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir) + download_dir = re.sub('\"', '', down_dir) + except EnvironmentError: + download_dir = os.getenv('XDG_DOWNLOAD_DIR') + + if not download_dir: + download_dir = os.path.expanduser('~/Downloads') + + # old + data_dir = os.path.expanduser('~/.lbrynet') + lbryum_dir = os.path.expanduser('~/.lbryum') + if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir): + return data_dir, lbryum_dir, download_dir + + # new + return user_data_dir('lbry/lbrynet'), user_data_dir('lbry/lbryum'), download_dir + + def server_port(server_and_port): server, port = server_and_port.split(':') return server, int(port) @@ -244,18 +214,15 @@ ADJUSTABLE_SETTINGS = { # will not be made automatically) 'auto_renew_claim_height_delta': (int, 0), 'cache_time': (int, 150), - 'data_dir': (str, default_data_dir), 'data_rate': (float, .0001), # points/megabyte 'delete_blobs_on_remove': (bool, True), 'dht_node_port': (int, 4444), - 'download_directory': (str, default_download_dir), 'download_timeout': (int, 180), 'download_mirrors': (list, ['blobs.lbry.io']), 'is_generous_host': (bool, True), 'announce_head_blobs_only': (bool, True), 'concurrent_announcers': (int, DEFAULT_CONCURRENT_ANNOUNCERS), 'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list, server_list_reverse), - 'lbryum_wallet_dir': (str, default_lbryum_dir), 'max_connections_per_stream': (int, 5), 'seek_head_blob_first': (bool, True), # TODO: writing json on the cmd line is a pain, come up with a nicer @@ -290,16 +257,36 @@ ADJUSTABLE_SETTINGS = { } -class Config: - def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None, - environment=None, cli_settings=None): +optional_str = typing.Optional[str] +class Config: + def __init__(self, fixed_defaults, adjustable_defaults: typing.Dict, persisted_settings=None, environment=None, + cli_settings=None, data_dir: optional_str = None, wallet_dir: optional_str = None, + download_dir: optional_str = None, file_name: optional_str = None): self._installation_id = None self._session_id = base58.b58encode(utils.generate_id()).decode() self._node_id = None self._fixed_defaults = fixed_defaults - self._adjustable_defaults = adjustable_defaults + + # copy the default adjustable settings + self._adjustable_defaults = {k: v for k, v in adjustable_defaults.items()} + + default_data_dir, default_wallet_dir, default_download_dir = None, None, None + # set the os specific default directories + if platform is WINDOWS: + default_data_dir, default_wallet_dir, default_download_dir = get_windows_directories() + elif platform is DARWIN: + default_data_dir, default_wallet_dir, default_download_dir = get_darwin_directories() + elif platform is LINUX: + default_data_dir, default_wallet_dir, default_download_dir = get_linux_directories() + else: + assert None not in [data_dir, wallet_dir, download_dir] + + self.data_dir = data_dir or default_data_dir + self.download_dir = download_dir or default_download_dir + self.wallet_dir = wallet_dir or default_wallet_dir + self.file_name = file_name or self.get_valid_settings_filename() self._data = { TYPE_DEFAULT: {}, # defaults @@ -483,16 +470,15 @@ class Config: } def save_conf_file_settings(self): - path = conf_file or self.get_valid_settings_filename() # reverse the conversions done after loading the settings from the conf # file rev = self._convert_conf_file_lists_reverse(self._data[TYPE_PERSISTED]) - ext = os.path.splitext(path)[1] + ext = os.path.splitext(self.file_name)[1] encoder = settings_encoders.get(ext, False) if not encoder: raise ValueError('Unknown settings format: {}. Available formats: {}' .format(ext, list(settings_encoders.keys()))) - with open(path, 'w') as settings_file: + with open(os.path.join(self.data_dir, self.file_name), 'w') as settings_file: settings_file.write(encoder(rev)) @staticmethod @@ -520,14 +506,13 @@ class Config: settings.node_id = settings.get_node_id() def load_conf_file_settings(self): - path = conf_file or self.get_valid_settings_filename() - self._read_conf_file(path) + self._read_conf_file(os.path.join(self.data_dir, self.file_name)) # initialize members depending on config file self.initialize_post_conf_load() def _read_conf_file(self, path): if not path or not os.path.exists(path): - return + raise FileNotFoundError(path) ext = os.path.splitext(path)[1] decoder = settings_decoders.get(ext, False) if not decoder: @@ -562,13 +547,15 @@ class Config: # although there is a risk of a race condition here we don't # expect there to be multiple processes accessing this # directory so the risk can be ignored - if not os.path.isdir(self['data_dir']): - os.makedirs(self['data_dir']) - return self['data_dir'] + if not os.path.isdir(self.data_dir): + os.makedirs(self.data_dir) + if not os.path.isdir(os.path.join(self.data_dir, "blobfiles")): + os.makedirs(os.path.join(self.data_dir, "blobfiles")) + return self.data_dir def ensure_wallet_dir(self): - if not os.path.isdir(self['lbryum_wallet_dir']): - os.makedirs(self['lbryum_wallet_dir']) + if not os.path.isdir(self.wallet_dir): + os.makedirs(self.wallet_dir) def get_log_filename(self): """ @@ -624,7 +611,7 @@ class Config: return self._session_id -settings = None # type: Config +settings: Config = None def get_default_env(): @@ -639,10 +626,15 @@ def get_default_env(): return Env(**env_defaults) -def initialize_settings(load_conf_file=True): +def initialize_settings(load_conf_file: typing.Optional[bool] = True, + data_dir: optional_str = None, wallet_dir: optional_str = None, + download_dir: optional_str = None): global settings if settings is None: settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS, - environment=get_default_env()) + environment=get_default_env(), data_dir=data_dir, wallet_dir=wallet_dir, + download_dir=download_dir) if load_conf_file: settings.load_conf_file_settings() + settings.ensure_data_dir() + settings.ensure_wallet_dir() diff --git a/lbrynet/extras/cli.py b/lbrynet/extras/cli.py index 1d45ddaee..16a8503f8 100644 --- a/lbrynet/extras/cli.py +++ b/lbrynet/extras/cli.py @@ -1,4 +1,16 @@ import sys +import os +import json +import asyncio +import argparse +import typing + +# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows +# This needs to happen before anything else +if 'win' in sys.platform: + import certifi + os.environ['SSL_CERT_FILE'] = certifi.where() + from twisted.internet import asyncioreactor if 'twisted.internet.reactor' not in sys.modules: asyncioreactor.install() @@ -10,28 +22,93 @@ else: # https://github.com/kivy/kivy/issues/4182 del sys.modules['twisted.internet.reactor'] asyncioreactor.install() - from twisted.internet import reactor - -import json -import asyncio +from twisted.internet import reactor +import logging from aiohttp.client_exceptions import ClientConnectorError from requests.exceptions import ConnectionError from docopt import docopt from textwrap import dedent -from lbrynet import __name__ as lbrynet_name +from lbrynet import conf, log_support, __name__ as lbrynet_name +from lbrynet.utils import check_connection, json_dumps_pretty from lbrynet.extras.daemon.Daemon import Daemon -from lbrynet.extras.daemon.DaemonControl import start as daemon_main from lbrynet.extras.daemon.DaemonConsole import main as daemon_console from lbrynet.extras.daemon.auth.client import LBRYAPIClient from lbrynet.extras.system_info import get_platform +log = logging.getLogger(lbrynet_name) -async def execute_command(method, params, conf_path=None): +optional_path_getter_type = typing.Optional[typing.Callable[[], str]] + + +def start_daemon(settings: typing.Optional[typing.Dict] = None, + console_output: typing.Optional[bool] = True, verbose: typing.Optional[typing.List[str]] = None, + data_dir: typing.Optional[str] = None, wallet_dir: typing.Optional[str] = None, + download_dir: typing.Optional[str] = None): + + settings = settings or {} + conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir) + for k, v in settings.items(): + conf.settings.update({k, v}, data_types=(conf.TYPE_CLI,)) + + log_support.configure_logging(conf.settings.get_log_filename(), console_output, verbose) + log_support.configure_loggly_handler() + log.debug('Final Settings: %s', conf.settings.get_current_settings_dict()) + log.info("Starting lbrynet-daemon from command line") + + if check_connection(): + daemon = Daemon() + daemon.start_listening() + reactor.run() + else: + log.info("Not connected to internet, unable to start") + + +def start_daemon_with_cli_args(argv=None, data_dir: typing.Optional[str] = None, + wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None): + parser = argparse.ArgumentParser() + parser.add_argument( + "--http-auth", dest="useauth", action="store_true", default=False + ) + parser.add_argument( + '--quiet', dest='quiet', action="store_true", + help='Disable all console output.' + ) + parser.add_argument( + '--verbose', nargs="*", + help=('Enable debug output. Optionally specify loggers for which debug output ' + 'should selectively be applied.') + ) + parser.add_argument( + '--version', action="store_true", + help='Show daemon version and quit' + ) + + args = parser.parse_args(argv) + settings = {} + if args.useauth: + settings['use_auth_http'] = True + + verbose = None + if args.verbose: + verbose = args.verbose + + console_output = not args.quiet + + if args.version: + print(json_dumps_pretty(get_platform())) + return + + return start_daemon(settings, console_output, verbose, data_dir, wallet_dir, download_dir) + + +async def execute_command(method, params, data_dir: typing.Optional[str] = None, + wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None): # this check if the daemon is running or not + conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir) api = None try: - api = await LBRYAPIClient.get_client(conf_path) + api = await LBRYAPIClient.get_client() await api.status() except (ClientConnectorError, ConnectionError): if api: @@ -119,14 +196,31 @@ def main(argv=None): print_help() return 1 - conf_path = None - if len(argv) and argv[0] == "--conf": + data_dir = None + if len(argv) and argv[0] == "--data_dir": if len(argv) < 2: - print("No config file specified for --conf option") + print("No directory specified for --data_dir option") print_help() return 1 + data_dir = argv[1] + argv = argv[2:] - conf_path = argv[1] + wallet_dir = None + if len(argv) and argv[0] == "--wallet_dir": + if len(argv) < 2: + print("No directory specified for --wallet_dir option") + print_help() + return 1 + wallet_dir = argv[1] + argv = argv[2:] + + download_dir = None + if len(argv) and argv[0] == "--download_dir": + if len(argv) < 2: + print("No directory specified for --data_dir option") + print_help() + return 1 + download_dir = argv[1] argv = argv[2:] method, args = argv[0], argv[1:] @@ -145,7 +239,7 @@ def main(argv=None): return 0 elif method == 'start': - sys.exit(daemon_main(args, conf_path)) + sys.exit(start_daemon_with_cli_args(args, data_dir, wallet_dir, download_dir)) elif method == 'console': sys.exit(daemon_console()) @@ -167,7 +261,7 @@ def main(argv=None): parsed = docopt(fn.__doc__, args) params = set_kwargs(parsed) loop = asyncio.get_event_loop() - loop.run_until_complete(execute_command(method, params, conf_path)) + loop.run_until_complete(execute_command(method, params, data_dir, wallet_dir, download_dir)) return 0 diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index 8c3f034d5..5514a4cae 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -59,62 +59,18 @@ def from_future(coroutine: asyncio.coroutine) -> defer.Deferred: return defer.Deferred.fromFuture(asyncio.ensure_future(coroutine)) -def get_wallet_config(): - wallet_type = GCS('wallet') - if wallet_type == conf.LBRYCRD_WALLET: - raise ValueError('LBRYcrd Wallet is no longer supported') - elif wallet_type != conf.LBRYUM_WALLET: - raise ValueError(f'Wallet Type {wallet_type} is not valid') - lbryum_servers = {address: {'t': str(port)} - for address, port in GCS('lbryum_servers')} - config = { - 'auto_connect': True, - 'chain': GCS('blockchain_name'), - 'default_servers': lbryum_servers - } - if 'use_keyring' in conf.settings: - config['use_keyring'] = GCS('use_keyring') - if conf.settings['lbryum_wallet_dir']: - config['lbryum_path'] = GCS('lbryum_wallet_dir') - return config - - -class ConfigSettings: - @staticmethod - def get_conf_setting(setting_name): - return conf.settings[setting_name] - - @staticmethod - def get_blobfiles_dir(): - if conf.settings['BLOBFILES_DIR'] == "blobfiles": - return os.path.join(GCS("data_dir"), "blobfiles") - else: - log.info("Using non-default blobfiles directory: %s", conf.settings['BLOBFILES_DIR']) - return conf.settings['BLOBFILES_DIR'] - - @staticmethod - def get_node_id(): - return conf.settings.node_id - - @staticmethod - @defer.inlineCallbacks - def get_external_ip(): # used if upnp is disabled or non-functioning - try: - buf = [] - response = yield treq.get("https://api.lbry.io/ip") - yield treq.collect(response, buf.append) - parsed = json.loads(b"".join(buf).decode()) - if parsed['success']: - return parsed['data']['ip'] - return - except Exception as err: - return - - - -# Shorthand for common ConfigSettings methods -CS = ConfigSettings -GCS = ConfigSettings.get_conf_setting +@defer.inlineCallbacks +def get_external_ip(): # used if upnp is disabled or non-functioning + try: + buf = [] + response = yield treq.get("https://api.lbry.io/ip") + yield treq.collect(response, buf.append) + parsed = json.loads(b"".join(buf).decode()) + if parsed['success']: + return parsed['data']['ip'] + return + except Exception as err: + return class DatabaseComponent(Component): @@ -146,18 +102,6 @@ class DatabaseComponent(Component): # check directories exist, create them if they don't log.info("Loading databases") - if not os.path.exists(GCS('download_directory')): - os.mkdir(GCS('download_directory')) - - if not os.path.exists(GCS('data_dir')): - os.mkdir(GCS('data_dir')) - self._write_db_revision_file(self.get_current_db_revision()) - log.debug("Created the db revision file: %s", self.get_revision_filename()) - - if not os.path.exists(CS.get_blobfiles_dir()): - os.mkdir(CS.get_blobfiles_dir()) - log.debug("Created the blobfile directory: %s", str(CS.get_blobfiles_dir())) - if not os.path.exists(self.get_revision_filename()): log.warning("db_revision file not found. Creating it") self._write_db_revision_file(self.get_current_db_revision()) @@ -174,13 +118,13 @@ class DatabaseComponent(Component): from lbrynet.extras.daemon.migrator import dbmigrator log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision()) yield threads.deferToThread( - dbmigrator.migrate_db, GCS('data_dir'), old_revision, self.get_current_db_revision() + dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision() ) self._write_db_revision_file(self.get_current_db_revision()) log.info("Finished upgrading the databases.") # start SQLiteStorage - self.storage = SQLiteStorage(GCS('data_dir')) + self.storage = SQLiteStorage(conf.settings.data_dir) yield self.storage.setup() @defer.inlineCallbacks @@ -198,9 +142,9 @@ class HeadersComponent(Component): def __init__(self, component_manager): super().__init__(component_manager) - self.headers_dir = os.path.join(conf.settings['lbryum_wallet_dir'], 'lbc_mainnet') + self.headers_dir = os.path.join(conf.settings.wallet_dir, 'lbc_mainnet') self.headers_file = os.path.join(self.headers_dir, 'headers') - self.old_file = os.path.join(conf.settings['lbryum_wallet_dir'], 'blockchain_headers') + self.old_file = os.path.join(conf.settings.wallet_dir, 'blockchain_headers') self._downloading_headers = None self._headers_progress_percent = 0 @@ -260,7 +204,7 @@ class HeadersComponent(Component): ledger = SimpleNamespace() ledger.config = { 'default_servers': conf.settings['lbryum_servers'], - 'data_path': conf.settings['lbryum_wallet_dir'] + 'data_path': conf.settings.wallet_dir } net = Network(ledger) first_connection = net.on_connected.first @@ -395,7 +339,7 @@ class BlobComponent(Component): dht_node = self.component_manager.get_component(DHT_COMPONENT) if dht_node: datastore = dht_node._dataStore - self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, datastore) + self.blob_manager = DiskBlobManager(os.path.join(conf.settings.data_dir, "blobfiles"), storage, datastore) return self.blob_manager.setup() def stop(self): @@ -428,34 +372,34 @@ class DHTComponent(Component): def get_status(self): return { - 'node_id': binascii.hexlify(CS.get_node_id()), + 'node_id': binascii.hexlify(conf.settings.get_node_id()), 'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts) } @defer.inlineCallbacks def start(self): self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT) - self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", GCS("peer_port")) - self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", GCS("dht_node_port")) - node_id = CS.get_node_id() + self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"]) + self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"]) + node_id = conf.settings.get_node_id() if node_id is None: node_id = generate_id() external_ip = self.upnp_component.external_ip if not external_ip: log.warning("UPnP component failed to get external ip") - external_ip = yield CS.get_external_ip() + external_ip = yield get_external_ip() if not external_ip: log.warning("failed to get external ip") self.dht_node = Node( node_id=node_id, - udpPort=GCS('dht_node_port'), + udpPort=conf.settings['dht_node_port'], externalUDPPort=self.external_udp_port, externalIP=external_ip, peerPort=self.external_peer_port ) - yield self.dht_node.start(GCS('known_dht_nodes'), block_on_join=False) + yield self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False) log.info("Started the dht") @defer.inlineCallbacks @@ -565,7 +509,7 @@ class FileManagerComponent(Component): blob_manager, storage, wallet, - GCS('download_directory') + conf.settings.download_dir ) yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory) @@ -598,7 +542,7 @@ class PeerProtocolServerComponent(Component): def start(self): wallet = self.component_manager.get_component(WALLET_COMPONENT) upnp = self.component_manager.get_component(UPNP_COMPONENT) - peer_port = GCS('peer_port') + peer_port = conf.settings['peer_port'] query_handlers = { handler.get_primary_query_identifier(): handler for handler in [ BlobRequestHandlerFactory( @@ -640,7 +584,7 @@ class ReflectorComponent(Component): def __init__(self, component_manager): super().__init__(component_manager) - self.reflector_server_port = GCS('reflector_port') + self.reflector_server_port = conf.settings['reflector_port'] self.reflector_server = None @property @@ -673,9 +617,9 @@ class UPnPComponent(Component): def __init__(self, component_manager): super().__init__(component_manager) - self._int_peer_port = GCS('peer_port') - self._int_dht_node_port = GCS('dht_node_port') - self.use_upnp = GCS('use_upnp') + self._int_peer_port = conf.settings['peer_port'] + self._int_dht_node_port = conf.settings['dht_node_port'] + self.use_upnp = conf.settings['use_upnp'] self.upnp = None self.upnp_redirects = {} self.external_ip = None @@ -719,7 +663,7 @@ class UPnPComponent(Component): if external_ip == "0.0.0.0" or not external_ip: log.warning("unable to get external ip from UPnP, checking lbry.io fallback") - external_ip = yield CS.get_external_ip() + external_ip = yield get_external_ip() if self.external_ip and self.external_ip != external_ip: log.info("external ip changed from %s to %s", self.external_ip, external_ip) self.external_ip = external_ip @@ -775,7 +719,7 @@ class UPnPComponent(Component): def start(self): log.info("detecting external ip") if not self.use_upnp: - self.external_ip = yield CS.get_external_ip() + self.external_ip = yield get_external_ip() return success = False yield self._maintain_redirects() diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index 46a50e09d..0b23ac3a7 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -166,9 +166,9 @@ def sort_claim_results(claims): def is_first_run(): if os.path.isfile(conf.settings.get_db_revision_filename()): return False - if os.path.isfile(os.path.join(conf.settings['data_dir'], 'lbrynet.sqlite')): + if os.path.isfile(os.path.join(conf.settings.data_dir, 'lbrynet.sqlite')): return False - if os.path.isfile(os.path.join(conf.settings['lbryum_wallet_dir'], 'blockchain_headers')): + if os.path.isfile(os.path.join(conf.settings.wallet_dir, 'blockchain_headers')): return False return True diff --git a/lbrynet/extras/daemon/DaemonControl.py b/lbrynet/extras/daemon/DaemonControl.py deleted file mode 100644 index 0ad0158fc..000000000 --- a/lbrynet/extras/daemon/DaemonControl.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import sys - -# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows -# This needs to happen before anything else -if 'win' in sys.platform: - import certifi - os.environ['SSL_CERT_FILE'] = certifi.where() - - -import argparse -import logging.handlers - -from twisted.internet import reactor -from lbrynet import utils, conf, log_support -from lbrynet.extras import system_info -from lbrynet.extras.daemon.Daemon import Daemon - -log = logging.getLogger(__name__) - - -def test_internet_connection(): - return utils.check_connection() - - -def start(argv=None, conf_path=None): - if conf_path is not None: - conf.conf_file = conf_path - - conf.initialize_settings() - - parser = argparse.ArgumentParser() - parser.add_argument( - "--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http'] - ) - parser.add_argument( - '--quiet', dest='quiet', action="store_true", - help='Disable all console output.' - ) - parser.add_argument( - '--verbose', nargs="*", - help=('Enable debug output. Optionally specify loggers for which debug output ' - 'should selectively be applied.') - ) - parser.add_argument( - '--version', action="store_true", - help='Show daemon version and quit' - ) - - args = parser.parse_args(argv) - if args.useauth: - conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,)) - - if args.version: - version = system_info.get_platform() - version['installation_id'] = conf.settings.installation_id - print(utils.json_dumps_pretty(version)) - return - - lbrynet_log = conf.settings.get_log_filename() - log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose) - log_support.configure_loggly_handler() - log.debug('Final Settings: %s', conf.settings.get_current_settings_dict()) - - log.info("Starting lbrynet-daemon from command line") - - if test_internet_connection(): - daemon = Daemon() - daemon.start_listening() - reactor.run() - else: - log.info("Not connected to internet, unable to start") diff --git a/lbrynet/extras/daemon/Downloader.py b/lbrynet/extras/daemon/Downloader.py index 45900e2f4..955cb900a 100644 --- a/lbrynet/extras/daemon/Downloader.py +++ b/lbrynet/extras/daemon/Downloader.py @@ -41,7 +41,7 @@ class GetStream: self.data_rate = data_rate or conf.settings['data_rate'] self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1] self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee'] - self.download_directory = conf.settings['download_directory'] + self.download_directory = conf.settings.download_dir self.timeout_counter = 0 self.code = None self.sd_hash = None diff --git a/lbrynet/extras/daemon/auth/client.py b/lbrynet/extras/daemon/auth/client.py index 50b0def8b..cea292038 100644 --- a/lbrynet/extras/daemon/auth/client.py +++ b/lbrynet/extras/daemon/auth/client.py @@ -125,9 +125,6 @@ class AuthAPIClient: class LBRYAPIClient: @staticmethod - def get_client(conf_path=None): - conf.conf_file = conf_path - if not conf.settings: - conf.initialize_settings() + def get_client(): return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \ UnAuthAPIClient.from_url(conf.settings.get_api_connection_string()) diff --git a/lbrynet/extras/daemon/auth/keyring.py b/lbrynet/extras/daemon/auth/keyring.py index eb4f7011a..727f3c6ef 100644 --- a/lbrynet/extras/daemon/auth/keyring.py +++ b/lbrynet/extras/daemon/auth/keyring.py @@ -73,8 +73,8 @@ class Keyring: @classmethod def load_from_disk(cls): - api_key_path = os.path.join(conf.settings['data_dir'], 'auth_token') - api_ssl_cert_path = os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem') + api_key_path = os.path.join(conf.settings.data_dir, 'auth_token') + api_ssl_cert_path = os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem') if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path): return with open(api_key_path, 'rb') as f: @@ -122,10 +122,10 @@ class Keyring: auth_token = APIKey.create(seed=None, name="api") - with open(os.path.join(conf.settings['data_dir'], 'auth_token'), 'wb') as f: + with open(os.path.join(conf.settings.data_dir, 'auth_token'), 'wb') as f: f.write(auth_token.secret.encode()) - with open(os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem'), 'wb') as f: + with open(os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem'), 'wb') as f: f.write(public_certificate.encode()) return cls(auth_token, public_certificate, private_certificate) diff --git a/lbrynet/extras/daemon/migrator/migrate5to6.py b/lbrynet/extras/daemon/migrator/migrate5to6.py index d8dcacdb5..6288830e5 100644 --- a/lbrynet/extras/daemon/migrator/migrate5to6.py +++ b/lbrynet/extras/daemon/migrator/migrate5to6.py @@ -7,8 +7,6 @@ from lbrynet.schema.decode import smart_decode log = logging.getLogger(__name__) -download_directory = conf.settings['download_directory'] - CREATE_TABLES_QUERY = """ pragma foreign_keys=on; pragma journal_mode=WAL; @@ -190,7 +188,7 @@ def do_migration(db_dir): # insert the file transaction.execute( "insert or ignore into file values (?, ?, ?, ?, ?)", - (stream_hash, stream_name, download_directory.encode('hex'), + (stream_hash, stream_name, conf.settings.download_dir.encode('hex'), data_rate, status) ) diff --git a/lbrynet/extras/wallet/manager.py b/lbrynet/extras/wallet/manager.py index 404c3c067..038e7dd87 100644 --- a/lbrynet/extras/wallet/manager.py +++ b/lbrynet/extras/wallet/manager.py @@ -171,12 +171,12 @@ class LbryWalletManager(BaseWalletManager): ledger_config = { 'auto_connect': True, 'default_servers': settings['lbryum_servers'], - 'data_path': settings['lbryum_wallet_dir'], + 'data_path': settings.wallet_dir, 'use_keyring': settings['use_keyring'], #'db': db } - wallets_directory = os.path.join(settings['lbryum_wallet_dir'], 'wallets') + wallets_directory = os.path.join(settings.wallet_dir, 'wallets') if not os.path.exists(wallets_directory): os.mkdir(wallets_directory) diff --git a/tests/integration/wallet/test_commands.py b/tests/integration/wallet/test_commands.py index 1039e9a4b..247347e8a 100644 --- a/tests/integration/wallet/test_commands.py +++ b/tests/integration/wallet/test_commands.py @@ -126,10 +126,10 @@ class CommandTestCase(IntegrationTestCase): logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY) lbry_conf.settings = None - lbry_conf.initialize_settings(load_conf_file=False) - lbry_conf.settings['data_dir'] = self.wallet_node.data_path - lbry_conf.settings['lbryum_wallet_dir'] = self.wallet_node.data_path - lbry_conf.settings['download_directory'] = self.wallet_node.data_path + lbry_conf.initialize_settings( + load_conf_file=False, data_dir=self.wallet_node.data_path, wallet_dir=self.wallet_node.data_path, + download_dir=self.wallet_node.data_path + ) lbry_conf.settings['use_upnp'] = False lbry_conf.settings['reflect_uploads'] = False lbry_conf.settings['blockchain_name'] = 'lbrycrd_regtest' diff --git a/tests/unit/test_conf.py b/tests/unit/test_conf.py index 5ab939b6d..29284a2dc 100644 --- a/tests/unit/test_conf.py +++ b/tests/unit/test_conf.py @@ -2,8 +2,10 @@ import os import json import sys import tempfile +import shutil from unittest import skipIf from twisted.trial import unittest +from twisted.internet import defer from lbrynet import conf from lbrynet.p2p.Error import InvalidCurrencyError @@ -15,11 +17,12 @@ class SettingsTest(unittest.TestCase): def tearDown(self): del os.environ['LBRY_TEST'] - @staticmethod - def get_mock_config_instance(): + def get_mock_config_instance(self): settings = {'test': (str, '')} env = conf.Env(**settings) - return conf.Config({}, settings, environment=env) + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(lambda : defer.succeed(shutil.rmtree(self.tmp_dir))) + return conf.Config({}, settings, environment=env, data_dir=self.tmp_dir, wallet_dir=self.tmp_dir, download_dir=self.tmp_dir) def test_envvar_is_read(self): settings = self.get_mock_config_instance() @@ -72,20 +75,19 @@ class SettingsTest(unittest.TestCase): out = settings.get('max_key_fee') self.assertEqual(out, valid_setting) - def test_data_dir(self): # check if these directories are returned as string and not unicode # otherwise there will be problems when calling os.path.join on # unicode directory names with string file names - self.assertEqual(str, type(conf.default_download_dir)) - self.assertEqual(str, type(conf.default_data_dir)) - self.assertEqual(str, type(conf.default_lbryum_dir)) + settings = conf.Config({}, {}) + self.assertEqual(str, type(settings.download_dir)) + self.assertEqual(str, type(settings.data_dir)) + self.assertEqual(str, type(settings.wallet_dir)) @skipIf('win' in sys.platform, 'fix me!') def test_load_save_config_file(self): # setup settings - adjustable_settings = {'data_dir': (str, conf.default_data_dir), - 'lbryum_servers': (list, [])} + adjustable_settings = {'lbryum_servers': (list, [])} env = conf.Env(**adjustable_settings) settings = conf.Config({}, adjustable_settings, environment=env) conf.settings = settings @@ -111,12 +113,14 @@ class SettingsTest(unittest.TestCase): settings = self.get_mock_config_instance() # nonexistent file - conf.conf_file = 'monkey.yml' + settings.file_name = 'monkey.yml' with self.assertRaises(FileNotFoundError): settings.load_conf_file_settings() # invalid extensions for filename in ('monkey.yymmll', 'monkey'): - conf.conf_file = filename + settings.file_name = filename + with open(os.path.join(self.tmp_dir, filename), "w"): + pass with self.assertRaises(ValueError): settings.load_conf_file_settings()