Merge pull request #1717 from lbryio/fix-directory-args

fix directory cli args and update help message

lbrynet/conf.py

@@ -9,7 +9,7 @@ import base58
 import yaml
 from appdirs import user_data_dir, user_config_dir
 from lbrynet import utils
-from lbrynet.p2p.Error import InvalidCurrencyError, NoSuchDirectoryError
+from lbrynet.p2p.Error import InvalidCurrencyError
 
 log = logging.getLogger(__name__)
 
@@ -196,6 +196,11 @@ FIXED_SETTINGS = {
 }
 
 ADJUSTABLE_SETTINGS = {
+    'data_dir': (str, ''),    # these blank defaults will be updated to OS specific defaults
+    'wallet_dir': (str, ''),
+    'lbryum_wallet_dir': (str, ''),  # to be deprecated
+    'download_directory': (str, ''),
+
     # By default, daemon will block all cross origin requests
     # but if this is set, this value will be used for the
     # Access-Control-Allow-Origin. For example
@@ -259,6 +264,7 @@ ADJUSTABLE_SETTINGS = {
 
 optional_str = typing.Optional[str]
 
+
 class Config:
     def __init__(self, fixed_defaults, adjustable_defaults: typing.Dict, persisted_settings=None, environment=None,
                  cli_settings=None, data_dir: optional_str = None, wallet_dir: optional_str = None,
@@ -272,21 +278,18 @@ class Config:
         # copy the default adjustable settings
         self._adjustable_defaults = {k: v for k, v in adjustable_defaults.items()}
 
-        default_data_dir, default_wallet_dir, default_download_dir = None, None, None
         # set the os specific default directories
         if platform is WINDOWS:
-            default_data_dir, default_wallet_dir, default_download_dir = get_windows_directories()
+            self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_windows_directories()
         elif platform is DARWIN:
-            default_data_dir, default_wallet_dir, default_download_dir = get_darwin_directories()
+            self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_darwin_directories()
         elif platform is LINUX:
-            default_data_dir, default_wallet_dir, default_download_dir = get_linux_directories()
+            self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_linux_directories()
         else:
             assert None not in [data_dir, wallet_dir, download_dir]
-
+            self.default_data_dir = data_dir
-        self.data_dir = data_dir or default_data_dir
+            self.default_wallet_dir = wallet_dir
-        self.download_dir = download_dir or default_download_dir
+            self.default_download_dir = download_dir
-        self.wallet_dir = wallet_dir or default_wallet_dir
-        self.file_name = file_name or self.get_valid_settings_filename()
 
         self._data = {
             TYPE_DEFAULT: {},  # defaults
@@ -323,6 +326,31 @@ class Config:
             cli_settings = {}
         self._validate_settings(cli_settings)
         self._data[TYPE_CLI].update(cli_settings)
+        self.file_name = file_name or 'daemon_settings.yml'
+
+    @property
+    def data_dir(self) -> optional_str:
+        data_dir = self.get('data_dir')
+        if not data_dir:
+            return
+        return os.path.expanduser(os.path.expandvars(data_dir))
+
+    @property
+    def download_dir(self) -> optional_str:
+        download_dir = self.get('download_directory')
+        if not download_dir:
+            return
+        return os.path.expanduser(os.path.expandvars(download_dir))
+
+    @property
+    def wallet_dir(self) -> optional_str:
+        if self.get('lbryum_wallet_dir') and not self.get('wallet_dir'):
+            log.warning("'lbryum_wallet_dir' setting will be deprecated, please update to 'wallet_dir'")
+            self['wallet_dir'] = self['lbryum_wallet_dir']
+        wallet_dir = self.get('wallet_dir')
+        if not wallet_dir:
+            return
+        return os.path.expanduser(os.path.expandvars(wallet_dir))
 
     def __repr__(self):
         return self.get_current_settings_dict().__repr__()
@@ -382,7 +410,7 @@ class Config:
         elif name == "download_directory":
             directory = str(value)
             if not os.path.exists(directory):
-                raise NoSuchDirectoryError(directory)
+                log.warning("download directory '%s' does not exist", directory)
 
     def is_default(self, name):
         """Check if a config value is wasn't specified by the user
@@ -506,7 +534,12 @@ class Config:
         settings.node_id = settings.get_node_id()
 
     def load_conf_file_settings(self):
-        self._read_conf_file(os.path.join(self.data_dir, self.file_name))
+        path = os.path.join(self.data_dir or self.default_data_dir, self.file_name)
+        if os.path.isfile(path):
+            self._read_conf_file(path)
+        self['data_dir'] = self.data_dir or self.default_data_dir
+        self['download_directory'] = self.download_dir or self.default_download_dir
+        self['wallet_dir'] = self.wallet_dir or self.default_wallet_dir
         # initialize members depending on config file
         self.initialize_post_conf_load()
 
@@ -576,13 +609,6 @@ class Config:
     def get_db_revision_filename(self):
         return os.path.join(self.ensure_data_dir(), self['DB_REVISION_FILE_NAME'])
 
-    def get_valid_settings_filename(self):
-        data_dir = self.ensure_data_dir()
-        json_path = os.path.join(data_dir, 'daemon_settings.json')
-        if os.path.isfile(json_path):
-            return json_path
-        return os.path.join(data_dir, 'daemon_settings.yml')
-
     def get_installation_id(self):
         install_id_filename = os.path.join(self.ensure_data_dir(), "install_id")
         if not self._installation_id:
@@ -636,5 +662,8 @@ def initialize_settings(load_conf_file: typing.Optional[bool] = True,
                           download_dir=download_dir)
         if load_conf_file:
             settings.load_conf_file_settings()
+        settings['data_dir'] = settings.data_dir or settings.default_data_dir
+        settings['download_directory'] = settings.download_dir or settings.default_download_dir
+        settings['wallet_dir'] = settings.wallet_dir or settings.default_wallet_dir
         settings.ensure_data_dir()
         settings.ensure_wallet_dir()

lbrynet/extras/cli.py

@@ -135,16 +135,26 @@ def print_help():
        lbrynet - LBRY command line client.
     
     USAGE
-       lbrynet [--conf <config file>] <command> [<args>]
+       lbrynet [--data_dir=<blob and database directory>] [--wallet_dir=<wallet directory>]
-    
+               [--download_dir=<downloads directory>] <command> [<args>]
+
     EXAMPLES
-        lbrynet start                  # starts the daemon. The daemon needs to be running for commands to work
+        lbrynet start                                # starts the daemon and listens for jsonrpc commands
-        lbrynet help                   # display this message
+        lbrynet help                                 # display this message
-        lbrynet help <command_name>    # get help for a command(doesn't need the daemon to be running)
+        lbrynet help <command_name>                  # get help for a command(doesn't need the daemon to be running)
-        lbrynet commands               # list available commands
+        lbrynet commands                             # list available commands
-        lbrynet status                 # get the running status of the daemon
+        lbrynet status                               # get the running status of the daemon
-        lbrynet --conf ~/l1.conf       # use ~/l1.conf as config file
+        lbrynet resolve what                         # resolve a name
-        lbrynet resolve what           # resolve a name
+
+        lbrynet --wallet_dir=~/wallet2 start         # start the daemon using an alternative wallet directory
+        lbrynet --data_dir=~/lbry start              # start the daemon using an alternative data directory
+
+        lbrynet --data_dir=~/lbry <command_name>     # run a command on a daemon using an alternative data directory,
+                                                     # which can contain a full daemon_settings.yml config file.
+                                                     # Note: since the daemon is what runs the wallet and
+                                                     # downloads files, only the --data_dir setting is needed when
+                                                     # running commands. The wallet_dir and download_dir would only
+                                                     # by used when starting the daemon.
     """))
 
 
@@ -196,32 +206,36 @@ def main(argv=None):
         print_help()
         return 1
 
-    data_dir = None
+    dir_args = {}
-    if len(argv) and argv[0] == "--data_dir":
+    if len(argv) > 2:
-        if len(argv) < 2:
+        dir_arg_keys = [
-            print("No directory specified for --data_dir option")
+            'data_dir',
-            print_help()
+            'wallet_dir',
-            return 1
+            'download_directory'
-        data_dir = argv[1]
+        ]
-        argv = argv[2:]
 
-    wallet_dir = None
+        for arg in argv:
-    if len(argv) and argv[0] == "--wallet_dir":
+            found_dir_arg = False
-        if len(argv) < 2:
+            for key in dir_arg_keys:
-            print("No directory specified for --wallet_dir option")
+                if arg.startswith(f'--{key}='):
-            print_help()
+                    if key in dir_args:
-            return 1
+                        print(f"Multiple values provided for '{key}' argument")
-        wallet_dir = argv[1]
+                        print_help()
-        argv = argv[2:]
+                        return 1
+                    dir_args[key] = os.path.expanduser(os.path.expandvars(arg.lstrip(f'--{key}=')))
+                    found_dir_arg = True
+            if not found_dir_arg:
+                break
+        argv = argv[len(dir_args):]
 
-    download_dir = None
+    data_dir = dir_args.get('data_dir')
-    if len(argv) and argv[0] == "--download_dir":
+    wallet_dir = dir_args.get('wallet_dir')
-        if len(argv) < 2:
+    download_dir = dir_args.get('download_directory')
-            print("No directory specified for --data_dir option")
+
-            print_help()
+    for k, v in dir_args.items():
+        if not os.path.isdir(v):
+            print(f"'{data_dir}' is not a directory, cannot use it for {k}")
             return 1
-        download_dir = argv[1]
-        argv = argv[2:]
 
     method, args = argv[0], argv[1:]
 

tests/mocks.py

@@ -502,6 +502,12 @@ def mock_conf_settings(obj, settings={}):
     conf.initialize_settings(False)
     original_settings = conf.settings
     conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS)
+    conf.settings['data_dir'] = settings.get('data_dir') or conf.settings.data_dir \
+                                or conf.settings.default_data_dir
+    conf.settings['download_directory'] = settings.get('download_directory') or conf.settings.download_dir \
+                                    or conf.settings.default_download_dir
+    conf.settings['wallet_dir'] = settings.get('wallet_dir') or conf.settings.wallet_dir or \
+                                  conf.settings.default_wallet_dir
     conf.settings.installation_id = conf.settings.get_installation_id()
     conf.settings.node_id = conf.settings.get_node_id()
     conf.settings.update(settings)

tests/unit/test_conf.py

@@ -112,11 +112,6 @@ class SettingsTest(unittest.TestCase):
     def test_load_file(self):
         settings = self.get_mock_config_instance()
 
-        # nonexistent file
-        settings.file_name = 'monkey.yml'
-        with self.assertRaises(FileNotFoundError):
-            settings.load_conf_file_settings()
-
         # invalid extensions
         for filename in ('monkey.yymmll', 'monkey'):
             settings.file_name = filename