updates, more refactoring for greater clarity

This commit is contained in:
Alex Grintsvayg 2017-01-17 12:29:09 -05:00
parent 267c6cbaca
commit cac8267e85
5 changed files with 154 additions and 68 deletions

View file

@ -77,7 +77,7 @@ class Env(envparse.Env):
my_schema = {
self._convert_key(key): self._convert_value(value)
for key, value in schema.items()
}
}
envparse.Env.__init__(self, **my_schema)
def __call__(self, key, *args, **kwargs):
@ -90,9 +90,10 @@ class Env(envparse.Env):
@staticmethod
def _convert_value(value):
"""
Allow value to be specified as a tuple or list. If you do this, the tuple/list
must be of the form (cast, default) or (cast, default, subcast)
""" Allow value to be specified as a tuple or list.
If you do this, the tuple/list must be of the
form (cast, default) or (cast, default, subcast)
"""
if isinstance(value, (tuple, list)):
new_value = {'cast': value[0], 'default': value[1]}
@ -101,6 +102,11 @@ class Env(envparse.Env):
return new_value
return value
TYPE_DEFAULT = 'default'
TYPE_PERSISTED = 'persisted'
TYPE_ENV = 'env'
TYPE_CLI = 'cli'
TYPE_RUNTIME = 'runtime'
FIXED_SETTINGS = {
'ANALYTICS_ENDPOINT': 'https://api.segment.io/v1',
@ -202,28 +208,52 @@ ADJUSTABLE_SETTINGS = {
}
class Config:
def __init__(self, fixed_defaults, adjustable_defaults, conf_file_settings={}, environment=None,
cli_settings={}):
class Config(object):
def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None,
environment=None, cli_settings=None):
self._lbry_id = None
self._session_id = base58.b58encode(utils.generate_id())
self.__fixed = fixed_defaults
self.__adjustable = adjustable_defaults
self._fixed_defaults = fixed_defaults
self._adjustable_defaults = adjustable_defaults
self._init_defaults()
self.conf_file_settings = conf_file_settings # from daemon_settings.yml
self.env_settings = self._parse_environment(environment) # from environment variables
self.cli_settings = cli_settings # from command_line flags/args
self.runtime_settings = {} # set by self.set() during runtime
self._assert_valid_settings()
self._combine_settings()
self._data = {
TYPE_DEFAULT: {}, # defaults
TYPE_PERSISTED: {}, # stored settings from daemon_settings.yml (or from a db, etc)
TYPE_ENV: {}, # settings from environment variables
TYPE_CLI: {}, # command-line arguments
TYPE_RUNTIME: {}, # set during runtime (using self.set(), etc)
}
# the order in which a piece of data is searched for. earlier types override later types
self._search_order = (
TYPE_RUNTIME, TYPE_CLI, TYPE_ENV, TYPE_PERSISTED, TYPE_DEFAULT
)
self._data[TYPE_DEFAULT].update(self._fixed_defaults)
self._data[TYPE_DEFAULT].update(
{k: v[1] for (k, v) in self._adjustable_defaults.iteritems()})
if persisted_settings is None:
persisted_settings = {}
self._validate_settings(persisted_settings)
self._data[TYPE_PERSISTED].update(persisted_settings)
env_settings = self._parse_environment(environment)
self._validate_settings(env_settings)
self._data[TYPE_ENV].update(env_settings)
if cli_settings is None:
cli_settings = {}
self._validate_settings(cli_settings)
self._data[TYPE_CLI].update(cli_settings)
def __repr__(self):
return self.combined_settings.__repr__()
return self.get_current_settings_dict().__repr__()
def __iter__(self):
for k in self.combined_settings.iterkeys():
for k in self._data[TYPE_DEFAULT].iterkeys():
yield k
def __getitem__(self, name):
@ -233,88 +263,118 @@ class Config:
return self.set(name, value)
def __contains__(self, name):
return name in self.combined_settings
return name in self._data[TYPE_DEFAULT]
def _parse_environment(self, environment):
@staticmethod
def _parse_environment(environment):
env_settings = {}
if environment is None:
environment = Env(**self.__adjustable)
if environment:
if environment is not None:
assert isinstance(environment, Env)
for opt in environment.original_schema:
env_settings[opt] = environment(opt)
return env_settings
def _init_defaults(self):
self.defaults = self.__fixed.copy()
for k, v in self.__adjustable.iteritems():
self.defaults[k] = v[1]
def _assert_valid_data_type(self, data_type):
assert data_type in self._data, KeyError('{} in is not a valid data type'.format(data_type))
def _assert_valid_settings(self):
for s in [self.conf_file_settings, self.env_settings, self.cli_settings,
self.runtime_settings]:
for name in s:
assert name in self.defaults, IndexError('%s is not a valid setting' % name)
def _is_valid_setting(self, name):
return name in self._data[TYPE_DEFAULT]
def _combine_settings(self):
self.combined_settings = {}
for s in [self.defaults, self.conf_file_settings, self.env_settings, self.cli_settings,
self.runtime_settings]:
self.combined_settings.update(s)
def _assert_valid_setting(self, name):
assert self._is_valid_setting(name), \
KeyError('{} in is not a valid setting'.format(name))
def get(self, name):
assert name in self.defaults, IndexError('%s is not a valid setting' % name)
return self.combined_settings[name]
def _validate_settings(self, data):
for name in data:
if not self._is_valid_setting(name):
raise KeyError('{} in is not a valid setting'.format(name))
def set(self, name, value, set_conf_setting=False):
assert name in self.defaults and name not in self.__fixed, KeyError
self.runtime_settings[name] = value
if set_conf_setting:
self.conf_file_settings[name] = value
self._combine_settings()
def _assert_editable_setting(self, name):
self._assert_valid_setting(name)
assert name not in self._fixed_defaults, \
ValueError('{} in is not an editable setting'.format(name))
def set_cli_settings(self, cli_settings):
self.cli_settings = cli_settings
self._assert_valid_settings()
self._combine_settings()
def get(self, name, data_type=None):
"""Get a config value
def update(self, updated_settings, set_conf_setting=False):
Args:
name: the name of the value to get
data_type: if given, get the value from a specific data set (see below)
Returns: the config value for the given name
If data_type is None, get() will search for the given name in each data set, in
order of precedence. It will return the first value it finds. This is the "effective"
value of a config name. For example, ENV values take precedence over DEFAULT values,
so if a value is present in ENV and in DEFAULT, the ENV value will be returned
"""
self._assert_valid_setting(name)
if data_type is not None:
self._assert_valid_data_type(data_type)
return self._data[data_type][name]
for data_type in self._search_order:
if name in self._data[data_type]:
return self._data[data_type][name]
raise KeyError('{} is not a valid setting'.format(name))
def set(self, name, value, data_types=(TYPE_RUNTIME,)):
"""Set a config value
Args:
name: the name of the value to set
value: the value
data_types: what type(s) of data this is
Returns: None
By default, this sets the RUNTIME value of a config. If you wish to set other
data types (e.g. PERSISTED values to save to a file, CLI values from parsed
command-line options, etc), you can specify that with the data_types param
"""
self._assert_editable_setting(name)
for data_type in data_types:
self._assert_valid_data_type(data_type)
self._data[data_type][name] = value
def update(self, updated_settings, data_types=(TYPE_RUNTIME,)):
for k, v in updated_settings.iteritems():
try:
self.set(k, v, set_conf_setting=set_conf_setting)
self.set(k, v, data_types=data_types)
except (KeyError, AssertionError):
pass
def get_current_settings_dict(self):
return self.combined_settings
current_settings = {}
for k, v in self._data[TYPE_DEFAULT].iteritems():
current_settings[k] = self.get(k)
return current_settings
def get_adjustable_settings_dict(self):
return {
opt: val for opt, val in self.get_current_settings_dict().iteritems()
if opt in self.__adjustable
}
if opt in self._adjustable_defaults
}
def save_conf_file_settings(self):
path = self.get_conf_filename()
ext = os.path.splitext(path)[1]
encoder = settings_encoders.get(ext, False)
assert encoder is not False, 'Unknown settings format .%s' % ext
assert encoder is not False, 'Unknown settings format %s' % ext
with open(path, 'w') as settings_file:
settings_file.write(encoder(self.conf_file_settings))
settings_file.write(encoder(self._data[TYPE_PERSISTED]))
def load_conf_file_settings(self):
path = self.get_conf_filename()
ext = os.path.splitext(path)[1]
decoder = settings_decoders.get(ext, False)
assert decoder is not False, 'Unknown settings format .%s' % ext
assert decoder is not False, 'Unknown settings format %s' % ext
try:
with open(path, 'r') as settings_file:
data = settings_file.read()
decoded = self._fix_old_conf_file_settings(decoder(data))
log.info('Loaded settings file: %s', path)
self.conf_file_settings.update(decoded)
self._assert_valid_settings()
self._combine_settings()
self._validate_settings(decoded)
self._data[TYPE_PERSISTED].update(decoded)
except (IOError, OSError) as err:
log.info('%s: Failed to update settings from %s', err, path)
@ -382,9 +442,14 @@ class Config:
settings = None
def get_default_env():
return Env(**ADJUSTABLE_SETTINGS)
def initialize_settings(load_conf_file=True):
global settings
if settings is None:
settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS)
settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS,
environment=get_default_env())
if load_conf_file:
settings.load_conf_file_settings()

View file

@ -603,11 +603,13 @@ class Daemon(AuthJSONRPCServer):
for key, setting_type in setting_types.iteritems():
if key in settings:
if can_update_key(settings, key, setting_type):
conf.settings.update({key: settings[key]}, set_conf_setting=True)
conf.settings.update({key: settings[key]},
data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED))
else:
try:
converted = setting_type(settings[key])
conf.settings.update({key: converted}, set_conf_setting=True)
conf.settings.update({key: converted},
data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED))
except Exception as err:
log.warning(err.message)
log.warning("error converting setting '%s' to type %s", key, setting_type)

View file

@ -110,7 +110,7 @@ def update_settings_from_args(args):
cli_settings['ui_branch'] = args.branch
cli_settings['use_auth_http'] = args.useauth
cli_settings['wallet'] = args.wallet
conf.settings.set_cli_settings(cli_settings)
conf.settings.update(cli_settings, data_types=(conf.TYPE_CLI,))
@defer.inlineCallbacks

View file

@ -209,7 +209,7 @@ create_stream_sd_file = {
def mock_conf_settings(obj, settings={}):
original_settings = conf.settings
conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS, environment=False)
conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS)
conf.settings.update(settings)
def _reset_settings():

View file

@ -14,8 +14,9 @@ class SettingsTest(unittest.TestCase):
@staticmethod
def get_mock_config_instance():
env = conf.Env(test=(str, ''))
return conf.Config({}, {'test': (str, '')}, environment=env)
settings = {'test': (str, '')}
env = conf.Env(**settings)
return conf.Config({}, settings, environment=env)
def test_envvar_is_read(self):
settings = self.get_mock_config_instance()
@ -35,3 +36,21 @@ class SettingsTest(unittest.TestCase):
settings = self.get_mock_config_instance()
setting_dict = settings.get_current_settings_dict()
self.assertEqual({'test': 'test_string'}, setting_dict)
def test_invalid_setting_raises_exception(self):
settings = self.get_mock_config_instance()
self.assertRaises(AssertionError, settings.set, 'invalid_name', 123)
def test_invalid_data_type_raises_exception(self):
settings = self.get_mock_config_instance()
self.assertIsNone(settings.set('test', 123))
self.assertRaises(AssertionError, settings.set, 'test', 123, ('fake_data_type',))
def test_setting_precedence(self):
settings = self.get_mock_config_instance()
settings.set('test', 'cli_test_string', data_types=(conf.TYPE_CLI,))
self.assertEqual('cli_test_string', settings['test'])
settings.set('test', 'this_should_not_take_precedence', data_types=(conf.TYPE_ENV,))
self.assertEqual('cli_test_string', settings['test'])
settings.set('test', 'runtime_takes_precedence', data_types=(conf.TYPE_RUNTIME,))
self.assertEqual('runtime_takes_precedence', settings['test'])