diff --git a/lbrynet/conf.py b/lbrynet/conf.py index ba61112ec..b06fa121a 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -77,7 +77,7 @@ class Env(envparse.Env): my_schema = { self._convert_key(key): self._convert_value(value) for key, value in schema.items() - } + } envparse.Env.__init__(self, **my_schema) def __call__(self, key, *args, **kwargs): @@ -90,9 +90,10 @@ class Env(envparse.Env): @staticmethod def _convert_value(value): - """ - Allow value to be specified as a tuple or list. If you do this, the tuple/list - must be of the form (cast, default) or (cast, default, subcast) + """ Allow value to be specified as a tuple or list. + + If you do this, the tuple/list must be of the + form (cast, default) or (cast, default, subcast) """ if isinstance(value, (tuple, list)): new_value = {'cast': value[0], 'default': value[1]} @@ -101,6 +102,11 @@ class Env(envparse.Env): return new_value return value +TYPE_DEFAULT = 'default' +TYPE_PERSISTED = 'persisted' +TYPE_ENV = 'env' +TYPE_CLI = 'cli' +TYPE_RUNTIME = 'runtime' FIXED_SETTINGS = { 'ANALYTICS_ENDPOINT': 'https://api.segment.io/v1', @@ -202,28 +208,52 @@ ADJUSTABLE_SETTINGS = { } -class Config: - def __init__(self, fixed_defaults, adjustable_defaults, conf_file_settings={}, environment=None, - cli_settings={}): +class Config(object): + def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None, + environment=None, cli_settings=None): + self._lbry_id = None self._session_id = base58.b58encode(utils.generate_id()) - self.__fixed = fixed_defaults - self.__adjustable = adjustable_defaults + self._fixed_defaults = fixed_defaults + self._adjustable_defaults = adjustable_defaults - self._init_defaults() - self.conf_file_settings = conf_file_settings # from daemon_settings.yml - self.env_settings = self._parse_environment(environment) # from environment variables - self.cli_settings = cli_settings # from command_line flags/args - self.runtime_settings = {} # set by self.set() during runtime - self._assert_valid_settings() - self._combine_settings() + self._data = { + TYPE_DEFAULT: {}, # defaults + TYPE_PERSISTED: {}, # stored settings from daemon_settings.yml (or from a db, etc) + TYPE_ENV: {}, # settings from environment variables + TYPE_CLI: {}, # command-line arguments + TYPE_RUNTIME: {}, # set during runtime (using self.set(), etc) + } + + # the order in which a piece of data is searched for. earlier types override later types + self._search_order = ( + TYPE_RUNTIME, TYPE_CLI, TYPE_ENV, TYPE_PERSISTED, TYPE_DEFAULT + ) + + self._data[TYPE_DEFAULT].update(self._fixed_defaults) + self._data[TYPE_DEFAULT].update( + {k: v[1] for (k, v) in self._adjustable_defaults.iteritems()}) + + if persisted_settings is None: + persisted_settings = {} + self._validate_settings(persisted_settings) + self._data[TYPE_PERSISTED].update(persisted_settings) + + env_settings = self._parse_environment(environment) + self._validate_settings(env_settings) + self._data[TYPE_ENV].update(env_settings) + + if cli_settings is None: + cli_settings = {} + self._validate_settings(cli_settings) + self._data[TYPE_CLI].update(cli_settings) def __repr__(self): - return self.combined_settings.__repr__() + return self.get_current_settings_dict().__repr__() def __iter__(self): - for k in self.combined_settings.iterkeys(): + for k in self._data[TYPE_DEFAULT].iterkeys(): yield k def __getitem__(self, name): @@ -233,88 +263,118 @@ class Config: return self.set(name, value) def __contains__(self, name): - return name in self.combined_settings + return name in self._data[TYPE_DEFAULT] - def _parse_environment(self, environment): + @staticmethod + def _parse_environment(environment): env_settings = {} - if environment is None: - environment = Env(**self.__adjustable) - if environment: + if environment is not None: assert isinstance(environment, Env) for opt in environment.original_schema: env_settings[opt] = environment(opt) return env_settings - def _init_defaults(self): - self.defaults = self.__fixed.copy() - for k, v in self.__adjustable.iteritems(): - self.defaults[k] = v[1] + def _assert_valid_data_type(self, data_type): + assert data_type in self._data, KeyError('{} in is not a valid data type'.format(data_type)) - def _assert_valid_settings(self): - for s in [self.conf_file_settings, self.env_settings, self.cli_settings, - self.runtime_settings]: - for name in s: - assert name in self.defaults, IndexError('%s is not a valid setting' % name) + def _is_valid_setting(self, name): + return name in self._data[TYPE_DEFAULT] - def _combine_settings(self): - self.combined_settings = {} - for s in [self.defaults, self.conf_file_settings, self.env_settings, self.cli_settings, - self.runtime_settings]: - self.combined_settings.update(s) + def _assert_valid_setting(self, name): + assert self._is_valid_setting(name), \ + KeyError('{} in is not a valid setting'.format(name)) - def get(self, name): - assert name in self.defaults, IndexError('%s is not a valid setting' % name) - return self.combined_settings[name] + def _validate_settings(self, data): + for name in data: + if not self._is_valid_setting(name): + raise KeyError('{} in is not a valid setting'.format(name)) - def set(self, name, value, set_conf_setting=False): - assert name in self.defaults and name not in self.__fixed, KeyError - self.runtime_settings[name] = value - if set_conf_setting: - self.conf_file_settings[name] = value - self._combine_settings() + def _assert_editable_setting(self, name): + self._assert_valid_setting(name) + assert name not in self._fixed_defaults, \ + ValueError('{} in is not an editable setting'.format(name)) - def set_cli_settings(self, cli_settings): - self.cli_settings = cli_settings - self._assert_valid_settings() - self._combine_settings() + def get(self, name, data_type=None): + """Get a config value - def update(self, updated_settings, set_conf_setting=False): + Args: + name: the name of the value to get + data_type: if given, get the value from a specific data set (see below) + + Returns: the config value for the given name + + If data_type is None, get() will search for the given name in each data set, in + order of precedence. It will return the first value it finds. This is the "effective" + value of a config name. For example, ENV values take precedence over DEFAULT values, + so if a value is present in ENV and in DEFAULT, the ENV value will be returned + """ + self._assert_valid_setting(name) + if data_type is not None: + self._assert_valid_data_type(data_type) + return self._data[data_type][name] + for data_type in self._search_order: + if name in self._data[data_type]: + return self._data[data_type][name] + raise KeyError('{} is not a valid setting'.format(name)) + + def set(self, name, value, data_types=(TYPE_RUNTIME,)): + """Set a config value + + Args: + name: the name of the value to set + value: the value + data_types: what type(s) of data this is + + Returns: None + + By default, this sets the RUNTIME value of a config. If you wish to set other + data types (e.g. PERSISTED values to save to a file, CLI values from parsed + command-line options, etc), you can specify that with the data_types param + """ + self._assert_editable_setting(name) + for data_type in data_types: + self._assert_valid_data_type(data_type) + self._data[data_type][name] = value + + def update(self, updated_settings, data_types=(TYPE_RUNTIME,)): for k, v in updated_settings.iteritems(): try: - self.set(k, v, set_conf_setting=set_conf_setting) + self.set(k, v, data_types=data_types) except (KeyError, AssertionError): pass def get_current_settings_dict(self): - return self.combined_settings + current_settings = {} + for k, v in self._data[TYPE_DEFAULT].iteritems(): + current_settings[k] = self.get(k) + return current_settings def get_adjustable_settings_dict(self): return { opt: val for opt, val in self.get_current_settings_dict().iteritems() - if opt in self.__adjustable - } + if opt in self._adjustable_defaults + } def save_conf_file_settings(self): path = self.get_conf_filename() ext = os.path.splitext(path)[1] encoder = settings_encoders.get(ext, False) - assert encoder is not False, 'Unknown settings format .%s' % ext + assert encoder is not False, 'Unknown settings format %s' % ext with open(path, 'w') as settings_file: - settings_file.write(encoder(self.conf_file_settings)) + settings_file.write(encoder(self._data[TYPE_PERSISTED])) def load_conf_file_settings(self): path = self.get_conf_filename() ext = os.path.splitext(path)[1] decoder = settings_decoders.get(ext, False) - assert decoder is not False, 'Unknown settings format .%s' % ext + assert decoder is not False, 'Unknown settings format %s' % ext try: with open(path, 'r') as settings_file: data = settings_file.read() decoded = self._fix_old_conf_file_settings(decoder(data)) log.info('Loaded settings file: %s', path) - self.conf_file_settings.update(decoded) - self._assert_valid_settings() - self._combine_settings() + self._validate_settings(decoded) + self._data[TYPE_PERSISTED].update(decoded) except (IOError, OSError) as err: log.info('%s: Failed to update settings from %s', err, path) @@ -382,9 +442,14 @@ class Config: settings = None +def get_default_env(): + return Env(**ADJUSTABLE_SETTINGS) + + def initialize_settings(load_conf_file=True): global settings if settings is None: - settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS) + settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS, + environment=get_default_env()) if load_conf_file: settings.load_conf_file_settings() diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index 405ae96ad..38ae68952 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -603,11 +603,13 @@ class Daemon(AuthJSONRPCServer): for key, setting_type in setting_types.iteritems(): if key in settings: if can_update_key(settings, key, setting_type): - conf.settings.update({key: settings[key]}, set_conf_setting=True) + conf.settings.update({key: settings[key]}, + data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED)) else: try: converted = setting_type(settings[key]) - conf.settings.update({key: converted}, set_conf_setting=True) + conf.settings.update({key: converted}, + data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED)) except Exception as err: log.warning(err.message) log.warning("error converting setting '%s' to type %s", key, setting_type) diff --git a/lbrynet/lbrynet_daemon/DaemonControl.py b/lbrynet/lbrynet_daemon/DaemonControl.py index cbd0edf56..89535fe3d 100644 --- a/lbrynet/lbrynet_daemon/DaemonControl.py +++ b/lbrynet/lbrynet_daemon/DaemonControl.py @@ -110,7 +110,7 @@ def update_settings_from_args(args): cli_settings['ui_branch'] = args.branch cli_settings['use_auth_http'] = args.useauth cli_settings['wallet'] = args.wallet - conf.settings.set_cli_settings(cli_settings) + conf.settings.update(cli_settings, data_types=(conf.TYPE_CLI,)) @defer.inlineCallbacks diff --git a/tests/mocks.py b/tests/mocks.py index 4e583c5a6..2ac30e175 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -209,7 +209,7 @@ create_stream_sd_file = { def mock_conf_settings(obj, settings={}): original_settings = conf.settings - conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS, environment=False) + conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS) conf.settings.update(settings) def _reset_settings(): diff --git a/tests/unit/test_conf.py b/tests/unit/test_conf.py index 91d74c211..2fce3e447 100644 --- a/tests/unit/test_conf.py +++ b/tests/unit/test_conf.py @@ -14,8 +14,9 @@ class SettingsTest(unittest.TestCase): @staticmethod def get_mock_config_instance(): - env = conf.Env(test=(str, '')) - return conf.Config({}, {'test': (str, '')}, environment=env) + settings = {'test': (str, '')} + env = conf.Env(**settings) + return conf.Config({}, settings, environment=env) def test_envvar_is_read(self): settings = self.get_mock_config_instance() @@ -35,3 +36,21 @@ class SettingsTest(unittest.TestCase): settings = self.get_mock_config_instance() setting_dict = settings.get_current_settings_dict() self.assertEqual({'test': 'test_string'}, setting_dict) + + def test_invalid_setting_raises_exception(self): + settings = self.get_mock_config_instance() + self.assertRaises(AssertionError, settings.set, 'invalid_name', 123) + + def test_invalid_data_type_raises_exception(self): + settings = self.get_mock_config_instance() + self.assertIsNone(settings.set('test', 123)) + self.assertRaises(AssertionError, settings.set, 'test', 123, ('fake_data_type',)) + + def test_setting_precedence(self): + settings = self.get_mock_config_instance() + settings.set('test', 'cli_test_string', data_types=(conf.TYPE_CLI,)) + self.assertEqual('cli_test_string', settings['test']) + settings.set('test', 'this_should_not_take_precedence', data_types=(conf.TYPE_ENV,)) + self.assertEqual('cli_test_string', settings['test']) + settings.set('test', 'runtime_takes_precedence', data_types=(conf.TYPE_RUNTIME,)) + self.assertEqual('runtime_takes_precedence', settings['test'])