updates, more refactoring for greater clarity

This commit is contained in:
Alex Grintsvayg 2017-01-17 12:29:09 -05:00
parent 267c6cbaca
commit cac8267e85
5 changed files with 154 additions and 68 deletions

View file

@ -90,9 +90,10 @@ class Env(envparse.Env):
@staticmethod @staticmethod
def _convert_value(value): def _convert_value(value):
""" """ Allow value to be specified as a tuple or list.
Allow value to be specified as a tuple or list. If you do this, the tuple/list
must be of the form (cast, default) or (cast, default, subcast) If you do this, the tuple/list must be of the
form (cast, default) or (cast, default, subcast)
""" """
if isinstance(value, (tuple, list)): if isinstance(value, (tuple, list)):
new_value = {'cast': value[0], 'default': value[1]} new_value = {'cast': value[0], 'default': value[1]}
@ -101,6 +102,11 @@ class Env(envparse.Env):
return new_value return new_value
return value return value
TYPE_DEFAULT = 'default'
TYPE_PERSISTED = 'persisted'
TYPE_ENV = 'env'
TYPE_CLI = 'cli'
TYPE_RUNTIME = 'runtime'
FIXED_SETTINGS = { FIXED_SETTINGS = {
'ANALYTICS_ENDPOINT': 'https://api.segment.io/v1', 'ANALYTICS_ENDPOINT': 'https://api.segment.io/v1',
@ -202,28 +208,52 @@ ADJUSTABLE_SETTINGS = {
} }
class Config: class Config(object):
def __init__(self, fixed_defaults, adjustable_defaults, conf_file_settings={}, environment=None, def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None,
cli_settings={}): environment=None, cli_settings=None):
self._lbry_id = None self._lbry_id = None
self._session_id = base58.b58encode(utils.generate_id()) self._session_id = base58.b58encode(utils.generate_id())
self.__fixed = fixed_defaults self._fixed_defaults = fixed_defaults
self.__adjustable = adjustable_defaults self._adjustable_defaults = adjustable_defaults
self._init_defaults() self._data = {
self.conf_file_settings = conf_file_settings # from daemon_settings.yml TYPE_DEFAULT: {}, # defaults
self.env_settings = self._parse_environment(environment) # from environment variables TYPE_PERSISTED: {}, # stored settings from daemon_settings.yml (or from a db, etc)
self.cli_settings = cli_settings # from command_line flags/args TYPE_ENV: {}, # settings from environment variables
self.runtime_settings = {} # set by self.set() during runtime TYPE_CLI: {}, # command-line arguments
self._assert_valid_settings() TYPE_RUNTIME: {}, # set during runtime (using self.set(), etc)
self._combine_settings() }
# the order in which a piece of data is searched for. earlier types override later types
self._search_order = (
TYPE_RUNTIME, TYPE_CLI, TYPE_ENV, TYPE_PERSISTED, TYPE_DEFAULT
)
self._data[TYPE_DEFAULT].update(self._fixed_defaults)
self._data[TYPE_DEFAULT].update(
{k: v[1] for (k, v) in self._adjustable_defaults.iteritems()})
if persisted_settings is None:
persisted_settings = {}
self._validate_settings(persisted_settings)
self._data[TYPE_PERSISTED].update(persisted_settings)
env_settings = self._parse_environment(environment)
self._validate_settings(env_settings)
self._data[TYPE_ENV].update(env_settings)
if cli_settings is None:
cli_settings = {}
self._validate_settings(cli_settings)
self._data[TYPE_CLI].update(cli_settings)
def __repr__(self): def __repr__(self):
return self.combined_settings.__repr__() return self.get_current_settings_dict().__repr__()
def __iter__(self): def __iter__(self):
for k in self.combined_settings.iterkeys(): for k in self._data[TYPE_DEFAULT].iterkeys():
yield k yield k
def __getitem__(self, name): def __getitem__(self, name):
@ -233,88 +263,118 @@ class Config:
return self.set(name, value) return self.set(name, value)
def __contains__(self, name): def __contains__(self, name):
return name in self.combined_settings return name in self._data[TYPE_DEFAULT]
def _parse_environment(self, environment): @staticmethod
def _parse_environment(environment):
env_settings = {} env_settings = {}
if environment is None: if environment is not None:
environment = Env(**self.__adjustable)
if environment:
assert isinstance(environment, Env) assert isinstance(environment, Env)
for opt in environment.original_schema: for opt in environment.original_schema:
env_settings[opt] = environment(opt) env_settings[opt] = environment(opt)
return env_settings return env_settings
def _init_defaults(self): def _assert_valid_data_type(self, data_type):
self.defaults = self.__fixed.copy() assert data_type in self._data, KeyError('{} in is not a valid data type'.format(data_type))
for k, v in self.__adjustable.iteritems():
self.defaults[k] = v[1]
def _assert_valid_settings(self): def _is_valid_setting(self, name):
for s in [self.conf_file_settings, self.env_settings, self.cli_settings, return name in self._data[TYPE_DEFAULT]
self.runtime_settings]:
for name in s:
assert name in self.defaults, IndexError('%s is not a valid setting' % name)
def _combine_settings(self): def _assert_valid_setting(self, name):
self.combined_settings = {} assert self._is_valid_setting(name), \
for s in [self.defaults, self.conf_file_settings, self.env_settings, self.cli_settings, KeyError('{} in is not a valid setting'.format(name))
self.runtime_settings]:
self.combined_settings.update(s)
def get(self, name): def _validate_settings(self, data):
assert name in self.defaults, IndexError('%s is not a valid setting' % name) for name in data:
return self.combined_settings[name] if not self._is_valid_setting(name):
raise KeyError('{} in is not a valid setting'.format(name))
def set(self, name, value, set_conf_setting=False): def _assert_editable_setting(self, name):
assert name in self.defaults and name not in self.__fixed, KeyError self._assert_valid_setting(name)
self.runtime_settings[name] = value assert name not in self._fixed_defaults, \
if set_conf_setting: ValueError('{} in is not an editable setting'.format(name))
self.conf_file_settings[name] = value
self._combine_settings()
def set_cli_settings(self, cli_settings): def get(self, name, data_type=None):
self.cli_settings = cli_settings """Get a config value
self._assert_valid_settings()
self._combine_settings()
def update(self, updated_settings, set_conf_setting=False): Args:
name: the name of the value to get
data_type: if given, get the value from a specific data set (see below)
Returns: the config value for the given name
If data_type is None, get() will search for the given name in each data set, in
order of precedence. It will return the first value it finds. This is the "effective"
value of a config name. For example, ENV values take precedence over DEFAULT values,
so if a value is present in ENV and in DEFAULT, the ENV value will be returned
"""
self._assert_valid_setting(name)
if data_type is not None:
self._assert_valid_data_type(data_type)
return self._data[data_type][name]
for data_type in self._search_order:
if name in self._data[data_type]:
return self._data[data_type][name]
raise KeyError('{} is not a valid setting'.format(name))
def set(self, name, value, data_types=(TYPE_RUNTIME,)):
"""Set a config value
Args:
name: the name of the value to set
value: the value
data_types: what type(s) of data this is
Returns: None
By default, this sets the RUNTIME value of a config. If you wish to set other
data types (e.g. PERSISTED values to save to a file, CLI values from parsed
command-line options, etc), you can specify that with the data_types param
"""
self._assert_editable_setting(name)
for data_type in data_types:
self._assert_valid_data_type(data_type)
self._data[data_type][name] = value
def update(self, updated_settings, data_types=(TYPE_RUNTIME,)):
for k, v in updated_settings.iteritems(): for k, v in updated_settings.iteritems():
try: try:
self.set(k, v, set_conf_setting=set_conf_setting) self.set(k, v, data_types=data_types)
except (KeyError, AssertionError): except (KeyError, AssertionError):
pass pass
def get_current_settings_dict(self): def get_current_settings_dict(self):
return self.combined_settings current_settings = {}
for k, v in self._data[TYPE_DEFAULT].iteritems():
current_settings[k] = self.get(k)
return current_settings
def get_adjustable_settings_dict(self): def get_adjustable_settings_dict(self):
return { return {
opt: val for opt, val in self.get_current_settings_dict().iteritems() opt: val for opt, val in self.get_current_settings_dict().iteritems()
if opt in self.__adjustable if opt in self._adjustable_defaults
} }
def save_conf_file_settings(self): def save_conf_file_settings(self):
path = self.get_conf_filename() path = self.get_conf_filename()
ext = os.path.splitext(path)[1] ext = os.path.splitext(path)[1]
encoder = settings_encoders.get(ext, False) encoder = settings_encoders.get(ext, False)
assert encoder is not False, 'Unknown settings format .%s' % ext assert encoder is not False, 'Unknown settings format %s' % ext
with open(path, 'w') as settings_file: with open(path, 'w') as settings_file:
settings_file.write(encoder(self.conf_file_settings)) settings_file.write(encoder(self._data[TYPE_PERSISTED]))
def load_conf_file_settings(self): def load_conf_file_settings(self):
path = self.get_conf_filename() path = self.get_conf_filename()
ext = os.path.splitext(path)[1] ext = os.path.splitext(path)[1]
decoder = settings_decoders.get(ext, False) decoder = settings_decoders.get(ext, False)
assert decoder is not False, 'Unknown settings format .%s' % ext assert decoder is not False, 'Unknown settings format %s' % ext
try: try:
with open(path, 'r') as settings_file: with open(path, 'r') as settings_file:
data = settings_file.read() data = settings_file.read()
decoded = self._fix_old_conf_file_settings(decoder(data)) decoded = self._fix_old_conf_file_settings(decoder(data))
log.info('Loaded settings file: %s', path) log.info('Loaded settings file: %s', path)
self.conf_file_settings.update(decoded) self._validate_settings(decoded)
self._assert_valid_settings() self._data[TYPE_PERSISTED].update(decoded)
self._combine_settings()
except (IOError, OSError) as err: except (IOError, OSError) as err:
log.info('%s: Failed to update settings from %s', err, path) log.info('%s: Failed to update settings from %s', err, path)
@ -382,9 +442,14 @@ class Config:
settings = None settings = None
def get_default_env():
return Env(**ADJUSTABLE_SETTINGS)
def initialize_settings(load_conf_file=True): def initialize_settings(load_conf_file=True):
global settings global settings
if settings is None: if settings is None:
settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS) settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS,
environment=get_default_env())
if load_conf_file: if load_conf_file:
settings.load_conf_file_settings() settings.load_conf_file_settings()

View file

@ -603,11 +603,13 @@ class Daemon(AuthJSONRPCServer):
for key, setting_type in setting_types.iteritems(): for key, setting_type in setting_types.iteritems():
if key in settings: if key in settings:
if can_update_key(settings, key, setting_type): if can_update_key(settings, key, setting_type):
conf.settings.update({key: settings[key]}, set_conf_setting=True) conf.settings.update({key: settings[key]},
data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED))
else: else:
try: try:
converted = setting_type(settings[key]) converted = setting_type(settings[key])
conf.settings.update({key: converted}, set_conf_setting=True) conf.settings.update({key: converted},
data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED))
except Exception as err: except Exception as err:
log.warning(err.message) log.warning(err.message)
log.warning("error converting setting '%s' to type %s", key, setting_type) log.warning("error converting setting '%s' to type %s", key, setting_type)

View file

@ -110,7 +110,7 @@ def update_settings_from_args(args):
cli_settings['ui_branch'] = args.branch cli_settings['ui_branch'] = args.branch
cli_settings['use_auth_http'] = args.useauth cli_settings['use_auth_http'] = args.useauth
cli_settings['wallet'] = args.wallet cli_settings['wallet'] = args.wallet
conf.settings.set_cli_settings(cli_settings) conf.settings.update(cli_settings, data_types=(conf.TYPE_CLI,))
@defer.inlineCallbacks @defer.inlineCallbacks

View file

@ -209,7 +209,7 @@ create_stream_sd_file = {
def mock_conf_settings(obj, settings={}): def mock_conf_settings(obj, settings={}):
original_settings = conf.settings original_settings = conf.settings
conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS, environment=False) conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS)
conf.settings.update(settings) conf.settings.update(settings)
def _reset_settings(): def _reset_settings():

View file

@ -14,8 +14,9 @@ class SettingsTest(unittest.TestCase):
@staticmethod @staticmethod
def get_mock_config_instance(): def get_mock_config_instance():
env = conf.Env(test=(str, '')) settings = {'test': (str, '')}
return conf.Config({}, {'test': (str, '')}, environment=env) env = conf.Env(**settings)
return conf.Config({}, settings, environment=env)
def test_envvar_is_read(self): def test_envvar_is_read(self):
settings = self.get_mock_config_instance() settings = self.get_mock_config_instance()
@ -35,3 +36,21 @@ class SettingsTest(unittest.TestCase):
settings = self.get_mock_config_instance() settings = self.get_mock_config_instance()
setting_dict = settings.get_current_settings_dict() setting_dict = settings.get_current_settings_dict()
self.assertEqual({'test': 'test_string'}, setting_dict) self.assertEqual({'test': 'test_string'}, setting_dict)
def test_invalid_setting_raises_exception(self):
settings = self.get_mock_config_instance()
self.assertRaises(AssertionError, settings.set, 'invalid_name', 123)
def test_invalid_data_type_raises_exception(self):
settings = self.get_mock_config_instance()
self.assertIsNone(settings.set('test', 123))
self.assertRaises(AssertionError, settings.set, 'test', 123, ('fake_data_type',))
def test_setting_precedence(self):
settings = self.get_mock_config_instance()
settings.set('test', 'cli_test_string', data_types=(conf.TYPE_CLI,))
self.assertEqual('cli_test_string', settings['test'])
settings.set('test', 'this_should_not_take_precedence', data_types=(conf.TYPE_ENV,))
self.assertEqual('cli_test_string', settings['test'])
settings.set('test', 'runtime_takes_precedence', data_types=(conf.TYPE_RUNTIME,))
self.assertEqual('runtime_takes_precedence', settings['test'])