From 02d2bad03fc5da4dd968821715e6b19d3847c2f9 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Tue, 15 Nov 2016 23:29:23 -0600 Subject: [PATCH 1/6] allow json to be used to set settings --- lbrynet/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 668e58769..ddb27a267 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -52,6 +52,8 @@ def convert_setting(env_val, current_val): def _convert_setting(env_val, current_val): + if isinstance(env_val, basestring): + return json.loads(env_val) new_type = env_val.__class__ current_type = current_val.__class__ if current_type is bool: From 7a76763610ab4e07d47f756dc1738088f8cdb818 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Tue, 15 Nov 2016 23:29:38 -0600 Subject: [PATCH 2/6] rename Setting to Settings --- lbrynet/conf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index ddb27a267..a2e106761 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -104,7 +104,7 @@ def add_env_settings_to_dict(settings_dict): return settings_dict -class Setting(object): +class Settings(object): """A collection of configuration settings""" __fixed = [] __excluded = ['get_dict', 'update'] @@ -139,7 +139,7 @@ class Setting(object): pass -class AdjustableSettings(Setting): +class AdjustableSettings(Settings): """Settings that are allowed to be overriden by the user""" def __init__(self): self.is_generous_host = True @@ -185,9 +185,10 @@ class AdjustableSettings(Setting): self.use_auth_http = False self.sd_download_timeout = 3 self.max_key_fee = {'USD': {'amount': 25.0, 'address': ''}} + Settings.__init__(self) -class ApplicationSettings(Setting): +class ApplicationSettings(Settings): """Settings that are constants and shouldn't be overriden""" def __init__(self): self.MAX_HANDSHAKE_SIZE = 64*KB @@ -215,6 +216,7 @@ class ApplicationSettings(Setting): self.LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv' self.ANALYTICS_ENDPOINT = 'https://api.segment.io/v1' self.ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H=' + Settings.__init__(self) APPLICATION_SETTINGS = AdjustableSettings() From b88e771eb2a3b69ca90707908fc96cd02073b379 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 16 Nov 2016 13:38:38 -0600 Subject: [PATCH 3/6] bug fix: use proper settings dictionary --- lbrynet/lbrynet_daemon/DaemonControl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/DaemonControl.py b/lbrynet/lbrynet_daemon/DaemonControl.py index b549d175a..fb955e34e 100644 --- a/lbrynet/lbrynet_daemon/DaemonControl.py +++ b/lbrynet/lbrynet_daemon/DaemonControl.py @@ -76,7 +76,7 @@ def start(): lbrynet_log = settings.get_log_filename() log_support.configure_logging(lbrynet_log, args.logtoconsole, args.verbose) - log.debug('Final Settings: %s', settings.__dict__) + log.debug('Final Settings: %s', settings.get_dict()) try: log.debug('Checking for an existing lbrynet daemon instance') From 464bd11c3a7982cc3b37c3cbe6d62cc3264e628e Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 16 Nov 2016 13:38:43 -0600 Subject: [PATCH 4/6] Improve parsing of environment variables Switch adjustable settings to use the envparse library. This provides more flexible and robust parsing. --- lbrynet/conf.py | 228 ++++++++++++++++++++++++------------------------ 1 file changed, 115 insertions(+), 113 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index a2e106761..a31ab06f6 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -6,6 +6,7 @@ import sys import yaml from appdirs import user_data_dir +import envparse LBRYCRD_WALLET = 'lbrycrd' LBRYUM_WALLET = 'lbryum' @@ -41,69 +42,6 @@ else: default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum") -def convert_setting(env_val, current_val): - try: - return _convert_setting(env_val, current_val) - except Exception as exc: - log.warning( - 'Failed to convert %s. Returning original: %s: %s', - env_val, current_val, exc) - return current_val - - -def _convert_setting(env_val, current_val): - if isinstance(env_val, basestring): - return json.loads(env_val) - new_type = env_val.__class__ - current_type = current_val.__class__ - if current_type is bool: - if new_type is bool: - return env_val - elif str(env_val).lower() == "false": - return False - elif str(env_val).lower() == "true": - return True - else: - raise ValueError('{} is not a valid boolean value'.format(env_val)) - elif current_type is int: - return int(env_val) - elif current_type is float: - return float(env_val) - elif current_type is str: - return str(env_val) - elif current_type is unicode: - return unicode(env_val) - elif current_type is dict: - return dict(env_val) - elif current_type is list: - return list(env_val) - elif current_type is tuple: - return tuple(env_val) - else: - raise ValueError('Type {} cannot be converted'.format(current_type)) - - -def convert_env_setting(setting, value): - try: - env_val = os.environ[setting] - except KeyError: - return value - else: - return convert_setting(env_val, value) - - -def get_env_settings(settings): - for setting, value in settings.iteritems(): - setting = 'LBRY_' + setting.upper() - yield convert_env_setting(setting, value) - - -def add_env_settings_to_dict(settings_dict): - for setting, env_setting in zip(settings_dict, get_env_settings(settings_dict)): - settings_dict.update({setting: env_setting}) - return settings_dict - - class Settings(object): """A collection of configuration settings""" __fixed = [] @@ -121,9 +59,7 @@ class Settings(object): def __setitem__(self, key, value): assert key in self and key not in self.__fixed, KeyError(key) - old_value = self[key] - new_value = convert_setting(value, old_value) - self.__dict__[key] = new_value + self.__dict__[key] = value def __contains__(self, item): return item in iter(self) @@ -139,54 +75,118 @@ class Settings(object): pass +class Env(envparse.Env): + """An Env parser that automatically namespaces the variables with LBRY""" + NAMESPACE = 'LBRY_' + def __init__(self, **schema): + self.original_schema = schema + my_schema = { + self._convert_key(key): self._convert_value(value) + for key, value in schema.items() + } + envparse.Env.__init__(self, **my_schema) + + def __call__(self, key, *args, **kwargs): + my_key = self._convert_key(key) + return super(Env, self).__call__(my_key, *args, **kwargs) + + def _convert_key(self, key): + return Env.NAMESPACE + key.upper() + + def _convert_value(self, value): + """Allow value to be specified as an object, tuple or dict + + if object or dict, follow default envparse rules, if tuple + it needs to be of the form (cast, default) or (cast, default, subcast) + """ + if isinstance(value, dict): + return value + if isinstance(value, (tuple, list)): + new_value = {'cast': value[0], 'default': value[1]} + if len(value) == 3: + new_value['subcast'] = value[2] + return new_value + return value + + +def server_port(server_port): + server, port = server_port.split(':') + return server, port + + +DEFAULT_DHT_NODES = [ + ('lbrynet1.lbry.io', 4444), + ('lbrynet2.lbry.io', 4444), + ('lbrynet3.lbry.io', 4444) +] + + +ENVIRONMENT = Env( + is_generous_host=(bool, True), + run_on_startup=(bool, False), + download_directory=(str, default_download_directory), + max_upload=(float, 0.0), + max_download=(float, 0.0), + upload_log=(bool, True), + delete_blobs_on_remove=(bool, True), + use_upnp=(bool, True), + start_lbrycrdd=(bool, True), + run_reflector_server=(bool, False), + startup_scripts=(list, []), + # TODO: this doesn't seem like the kind of thing that should + # be configured; move it elsewhere. + last_version=(dict, {'lbrynet': '0.0.1', 'lbryum': '0.0.1'}), + peer_port=(int, 3333), + dht_node_port=(int, 4444), + reflector_port=(int, 5566), + download_timeout=(int, 30), + max_search_results=(int, 25), + search_timeout=(float, 3.0), + cache_time=(int, 150), + host_ui=(bool, True), + check_ui_requirements=(bool, True), + local_ui_path=(bool, False), + api_port=(int, 5279), + search_servers=(list, ['lighthouse1.lbry.io:50005']), + data_rate=(float, .0001), # points/megabyte + min_info_rate=(float, .02), # points/1000 infos + min_valuable_info_rate=(float, .05), # points/1000 infos + min_valuable_hash_rate=(float, .05), # points/1000 infos + max_connections_per_stream=(int, 5), + known_dht_nodes=(list, DEFAULT_DHT_NODES, server_port), + pointtrader_server=(str, 'http://127.0.0.1:2424'), + reflector_servers=(list, [("reflector.lbry.io", 5566)], server_port), + wallet=(str, LBRYUM_WALLET), + ui_branch=(str, "master"), + default_ui_branch=(str, 'master'), + data_dir=(str, default_data_dir), + lbryum_wallet_dir=(str, default_lbryum_dir), + use_auth_http=(bool, False), + sd_download_timeout=(int, 3), + # TODO: document what the 'address' field is used for + # TODO: writing json on the cmd line is a pain, come up with a nicer + # parser for this data structure. (maybe MAX_KEY_FEE=USD:25 + max_key_fee=(json.loads, {'USD': {'amount': 25.0, 'address': ''}}) +) + + class AdjustableSettings(Settings): """Settings that are allowed to be overriden by the user""" - def __init__(self): - self.is_generous_host = True - self.run_on_startup = False - self.download_directory = default_download_directory - self.max_upload = 0.0 - self.max_download = 0.0 - self.upload_log = True - self.delete_blobs_on_remove = True - self.use_upnp = True - self.start_lbrycrdd = True - self.run_reflector_server = False - self.startup_scripts = [] - self.last_version = {'lbrynet': '0.0.1', 'lbryum': '0.0.1'} - self.peer_port = 3333 - self.dht_node_port = 4444 - self.reflector_port = 5566 - self.download_timeout = 30 - self.max_search_results = 25 - self.search_timeout = 3.0 - self.cache_time = 150 - self.host_ui = True - self.check_ui_requirements = True - self.local_ui_path = False - self.api_port = 5279 - self.data_rate = .0001 # points/megabyte - self.min_info_rate = .02 # points/1000 infos - self.min_valuable_info_rate = .05 # points/1000 infos - self.min_valuable_hash_rate = .05 # points/1000 infos - self.max_connections_per_stream = 5 - self.known_dht_nodes = [ - ('lbrynet1.lbry.io', 4444), - ('lbrynet2.lbry.io', 4444), - ('lbrynet3.lbry.io', 4444) - ] - self.pointtrader_server = 'http://127.0.0.1:2424' - self.reflector_servers = [("reflector.lbry.io", 5566)] - self.wallet = LBRYUM_WALLET - self.ui_branch = "master" - self.default_ui_branch = 'master' - self.data_dir = default_data_dir - self.lbryum_wallet_dir = default_lbryum_dir - self.use_auth_http = False - self.sd_download_timeout = 3 - self.max_key_fee = {'USD': {'amount': 25.0, 'address': ''}} + def __init__(self, environ=None): + self.environ = environ or ENVIRONMENT Settings.__init__(self) + def __getattr__(self, attr): + if attr in self.environ.original_schema: + return self.environ(attr) + raise AttributeError + + def get_dict(self): + return { + name: self.environ(name) + for name in self.environ.original_schema + } + class ApplicationSettings(Settings): """Settings that are constants and shouldn't be overriden""" @@ -230,6 +230,11 @@ class DefaultSettings(ApplicationSettings, AdjustableSettings): ApplicationSettings.__init__(self) AdjustableSettings.__init__(self) + def get_dict(self): + d = ApplicationSettings.get_dict(self) + d.update(AdjustableSettings.get_dict(self)) + return d + DEFAULT_SETTINGS = DefaultSettings() @@ -237,9 +242,6 @@ DEFAULT_SETTINGS = DefaultSettings() class Config(DefaultSettings): __shared_state = copy.deepcopy(DEFAULT_SETTINGS.get_dict()) - def __init__(self): - self.__dict__ = add_env_settings_to_dict(self.__shared_state) - @property def ORIGIN(self): return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port) From 0bbc8f5b7eb858437bbd9322578b5ce9675dbb28 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 16 Nov 2016 13:56:33 -0600 Subject: [PATCH 5/6] add envparse to requirements --- packaging/windows/init.ps1 | 1 + requirements.txt | 3 ++- setup.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packaging/windows/init.ps1 b/packaging/windows/init.ps1 index 3d9b1e7e7..eb4e27d98 100644 --- a/packaging/windows/init.ps1 +++ b/packaging/windows/init.ps1 @@ -55,6 +55,7 @@ C:\Python27\Scripts\pip.exe install colorama==0.3.7 C:\Python27\Scripts\pip.exe install dnspython==1.12.0 C:\Python27\Scripts\pip.exe install ecdsa==0.13 +C:\Python27\Scripts\pip.exe install envparse==0.2.0 C:\Python27\Scripts\pip.exe install jsonrpc==1.2 diff --git a/requirements.txt b/requirements.txt index b5f035da4..79b24db86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ argparse==1.2.1 colorama==0.3.7 dnspython==1.12.0 ecdsa==0.13 +envparse==0.2.0 gmpy==1.17 jsonrpc==1.2 jsonrpclib==0.1.7 @@ -30,4 +31,4 @@ zope.interface==4.1.3 base58==0.2.2 googlefinance==0.7 pyyaml==3.12 -service_identity==16.0.0 \ No newline at end of file +service_identity==16.0.0 diff --git a/setup.py b/setup.py index 5182359e3..96d8ea6d9 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,8 @@ requires = [ 'base58', 'googlefinance', 'requests_futures', - 'PyYAML' + 'PyYAML', + 'envparse' ] console_scripts = [ @@ -251,6 +252,7 @@ elif platform == WINDOWS: 'cx_Freeze', 'dns', 'ecdsa', + 'envparse', 'gmpy', 'googlefinance', 'jsonrpc', From 91fb78b07798f9e531b16b9f73faeca7f7d4f40b Mon Sep 17 00:00:00 2001 From: Job Evers Date: Tue, 22 Nov 2016 11:50:54 -0600 Subject: [PATCH 6/6] Update TODO with more useful info --- lbrynet/conf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index a31ab06f6..d593c51f5 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -163,7 +163,11 @@ ENVIRONMENT = Env( lbryum_wallet_dir=(str, default_lbryum_dir), use_auth_http=(bool, False), sd_download_timeout=(int, 3), - # TODO: document what the 'address' field is used for + # TODO: this field is more complicated than it needs to be because + # it goes through a Fee validator when loaded by the exchange rate + # manager. Look into refactoring the exchange rate conversion to + # take in a simpler form. + # # TODO: writing json on the cmd line is a pain, come up with a nicer # parser for this data structure. (maybe MAX_KEY_FEE=USD:25 max_key_fee=(json.loads, {'USD': {'amount': 25.0, 'address': ''}})