import os import sys import typing import logging from argparse import ArgumentParser from contextlib import contextmanager from typing import Tuple import yaml from appdirs import user_data_dir, user_download_dir from lbry.error import InvalidCurrencyError from lbry.dht import constants from lbry.wallet.coinselection import COIN_SELECTION_STRATEGIES log = logging.getLogger(__name__) NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name T = typing.TypeVar('T') CURRENCIES = { 'BTC': {'type': 'crypto'}, 'LBC': {'type': 'crypto'}, 'USD': {'type': 'fiat'}, } class Setting(typing.Generic[T]): def __init__(self, doc: str, default: typing.Optional[T] = None, previous_names: typing.Optional[typing.List[str]] = None, metavar: typing.Optional[str] = None): self.doc = doc self.default = default self.previous_names = previous_names or [] self.metavar = metavar def __set_name__(self, owner, name): self.name = name # pylint: disable=attribute-defined-outside-init @property def cli_name(self): return f"--{self.name.replace('_', '-')}" @property def no_cli_name(self): return f"--no-{self.name.replace('_', '-')}" def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T: if obj is None: return self for location in obj.search_order: if self.name in location: return location[self.name] return self.default def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]): if val == NOT_SET: for location in obj.modify_order: if self.name in location: del location[self.name] else: self.validate(val) for location in obj.modify_order: location[self.name] = val def validate(self, value): raise NotImplementedError() def deserialize(self, value): # pylint: disable=no-self-use return value def serialize(self, value): # pylint: disable=no-self-use return value def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, metavar=self.metavar, default=NOT_SET ) class String(Setting[str]): def validate(self, value): assert isinstance(value, str), \ f"Setting '{self.name}' must be a string." # TODO: removes this after pylint starts to understand generics def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation return super().__get__(obj, owner) class Integer(Setting[int]): def validate(self, value): assert isinstance(value, int), \ f"Setting '{self.name}' must be an integer." def deserialize(self, value): return int(value) class Float(Setting[float]): def validate(self, value): assert isinstance(value, float), \ f"Setting '{self.name}' must be a decimal." def deserialize(self, value): return float(value) class Toggle(Setting[bool]): def validate(self, value): assert isinstance(value, bool), \ f"Setting '{self.name}' must be a true/false value." def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, action="store_true", default=NOT_SET ) parser.add_argument( self.no_cli_name, help=f"Opposite of {self.cli_name}", dest=self.name, action="store_false", default=NOT_SET ) class Path(String): def __init__(self, doc: str, *args, default: str = '', **kwargs): super().__init__(doc, default, *args, **kwargs) def __get__(self, obj, owner) -> str: value = super().__get__(obj, owner) if isinstance(value, str): return os.path.expanduser(os.path.expandvars(value)) return value class MaxKeyFee(Setting[dict]): def validate(self, value): if value is not None: assert isinstance(value, dict) and set(value) == {'currency', 'amount'}, \ f"Setting '{self.name}' must be a dict like \"{{'amount': 50.0, 'currency': 'USD'}}\"." if value["currency"] not in CURRENCIES: raise InvalidCurrencyError(value["currency"]) @staticmethod def _parse_list(l): if l == ['null']: return None assert len(l) == 2, ( 'Max key fee is made up of either two values: ' '"AMOUNT CURRENCY", or "null" (to set no limit)' ) try: amount = float(l[0]) except ValueError: raise AssertionError('First value in max key fee is a decimal: "AMOUNT CURRENCY"') currency = str(l[1]).upper() if currency not in CURRENCIES: raise InvalidCurrencyError(currency) return {'amount': amount, 'currency': currency} def deserialize(self, value): if value is None: return if isinstance(value, dict): return { 'currency': value['currency'], 'amount': float(value['amount']), } if isinstance(value, str): value = value.split() if isinstance(value, list): return self._parse_list(value) raise AssertionError('Invalid max key fee.') def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, nargs='+', metavar=('AMOUNT', 'CURRENCY'), default=NOT_SET ) parser.add_argument( self.no_cli_name, help=f"Disable maximum key fee check.", dest=self.name, const=None, action="store_const", default=NOT_SET ) class StringChoice(String): def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs): super().__init__(doc, default, *args, **kwargs) if not valid_values: raise ValueError("No valid values provided") if default not in valid_values: raise ValueError(f"Default value must be one of: {', '.join(valid_values)}") self.valid_values = valid_values def validate(self, value): super().validate(value) if value not in self.valid_values: raise ValueError(f"Setting '{self.name}' value must be one of: {', '.join(self.valid_values)}") class ListSetting(Setting[list]): def validate(self, value): assert isinstance(value, (tuple, list)), \ f"Setting '{self.name}' must be a tuple or list." def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, action='append' ) class Servers(ListSetting): def validate(self, value): assert isinstance(value, (tuple, list)), \ f"Setting '{self.name}' must be a tuple or list of servers." for idx, server in enumerate(value): assert isinstance(server, (tuple, list)) and len(server) == 2, \ f"Server defined '{server}' at index {idx} in setting " \ f"'{self.name}' must be a tuple or list of two items." assert isinstance(server[0], str), \ f"Server defined '{server}' at index {idx} in setting " \ f"'{self.name}' must be have hostname as string in first position." assert isinstance(server[1], int), \ f"Server defined '{server}' at index {idx} in setting " \ f"'{self.name}' must be have port as int in second position." def deserialize(self, value): servers = [] if isinstance(value, list): for server in value: if isinstance(server, str) and server.count(':') == 1: host, port = server.split(':') try: servers.append((host, int(port))) except ValueError: pass return servers def serialize(self, value): if value: return [f"{host}:{port}" for host, port in value] return value class Strings(ListSetting): def validate(self, value): assert isinstance(value, (tuple, list)), \ f"Setting '{self.name}' must be a tuple or list of strings." for idx, string in enumerate(value): assert isinstance(string, str), \ f"Value of '{string}' at index {idx} in setting " \ f"'{self.name}' must be a string." class EnvironmentAccess: PREFIX = 'LBRY_' def __init__(self, environ: dict): self.environ = environ def __contains__(self, item: str): return f'{self.PREFIX}{item.upper()}' in self.environ def __getitem__(self, item: str): return self.environ[f'{self.PREFIX}{item.upper()}'] class ArgumentAccess: def __init__(self, config: 'BaseConfig', args: dict): self.configuration = config self.args = {} if args: self.load(args) def load(self, args): for setting in self.configuration.get_settings(): value = getattr(args, setting.name, NOT_SET) if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None): self.args[setting.name] = setting.deserialize(value) def __contains__(self, item: str): return item in self.args def __getitem__(self, item: str): return self.args[item] class ConfigFileAccess: def __init__(self, config: 'BaseConfig', path: str): self.configuration = config self.path = path self.data = {} if self.exists: self.load() @property def exists(self): return self.path and os.path.exists(self.path) def load(self): cls = type(self.configuration) with open(self.path, 'r') as config_file: raw = config_file.read() serialized = yaml.load(raw) or {} for key, value in serialized.items(): attr = getattr(cls, key, None) if attr is None: for setting in self.configuration.settings: if key in setting.previous_names: attr = setting break if attr is not None: self.data[key] = attr.deserialize(value) def save(self): cls = type(self.configuration) serialized = {} for key, value in self.data.items(): attr = getattr(cls, key) serialized[key] = attr.serialize(value) with open(self.path, 'w') as config_file: config_file.write(yaml.safe_dump(serialized, default_flow_style=False)) def upgrade(self) -> bool: upgraded = False for key in list(self.data): for setting in self.configuration.settings: if key in setting.previous_names: self.data[setting.name] = self.data[key] del self.data[key] upgraded = True break return upgraded def __contains__(self, item: str): return item in self.data def __getitem__(self, item: str): return self.data[item] def __setitem__(self, key, value): self.data[key] = value def __delitem__(self, key): del self.data[key] TBC = typing.TypeVar('TBC', bound='BaseConfig') class BaseConfig: config = Path("Path to configuration file.", metavar='FILE') def __init__(self, **kwargs): self.runtime = {} # set internally or by various API calls self.arguments = {} # from command line arguments self.environment = {} # from environment variables self.persisted = {} # from config file self._updating_config = False self.set(**kwargs) def set(self, **kwargs): for key, value in kwargs.items(): setattr(self, key, value) return self @contextmanager def update_config(self): self._updating_config = True yield self self._updating_config = False if isinstance(self.persisted, ConfigFileAccess): self.persisted.save() @property def modify_order(self): locations = [self.runtime] if self._updating_config: locations.append(self.persisted) return locations @property def search_order(self): return [ self.runtime, self.arguments, self.environment, self.persisted ] @classmethod def get_settings(cls): for attr in dir(cls): setting = getattr(cls, attr) if isinstance(setting, Setting): yield setting @property def settings(self): return self.get_settings() @property def settings_dict(self): return { setting.name: getattr(self, setting.name) for setting in self.settings } @classmethod def create_from_arguments(cls, args) -> TBC: conf = cls() conf.set_arguments(args) conf.set_environment() conf.set_persisted() return conf @classmethod def contribute_to_argparse(cls, parser: ArgumentParser): for setting in cls.get_settings(): setting.contribute_to_argparse(parser) def set_arguments(self, args): self.arguments = ArgumentAccess(self, args) def set_environment(self, environ=None): self.environment = EnvironmentAccess(environ or os.environ) def set_persisted(self, config_file_path=None): if config_file_path is None: config_file_path = self.config if not config_file_path: return ext = os.path.splitext(config_file_path)[1] assert ext in ('.yml', '.yaml'),\ f"File extension '{ext}' is not supported, " \ f"configuration file must be in YAML (.yaml)." self.persisted = ConfigFileAccess(self, config_file_path) if self.persisted.upgrade(): self.persisted.save() class TranscodeConfig(BaseConfig): ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. ' f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}', '', previous_names=['ffmpeg_folder']) video_encoder = String('FFmpeg codec and parameters for the video encoding. ' 'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental', 'libx264 -crf 21 -preset faster -pix_fmt yuv420p') video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000) video_scaler = String('FFmpeg scaling parameters for reducing bitrate. ' 'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M', r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" ' r'-maxrate 8400K -bufsize 5000K') audio_encoder = String('FFmpeg codec and parameters for the audio encoding. ' 'Example: libopus -b:a 128k', 'aac -b:a 160k') volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm') volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240) class CLIConfig(TranscodeConfig): api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT') @property def api_connection_url(self) -> str: return f"http://{self.api}/lbryapi" @property def api_host(self): return self.api.split(':')[0] @property def api_port(self): return int(self.api.split(':')[1]) class Config(CLIConfig): db_url = String("Database connection URL, uses a local file based SQLite by default.") processes = Integer( "Multiprocessing, specify number of processes lbrynet can start (including main process)." " (-1: threads only, 0: equal to number of CPUs, >1: specific number of processes)", -1 ) console = StringChoice( "Basic text console output or advanced colored output with progress bars.", ["basic", "advanced"], "advanced" ) # directories download_dir = Path("Directory to store downloaded files.", metavar='DIR') data_dir = Path("Main directory containing blobs, wallets and blockchain data.", metavar='DIR') blob_dir = Path("Directory to store blobs (default: 'data_dir'/blobs).", metavar='DIR') wallet_dir = Path("Directory to store wallets (default: 'data_dir'/wallets).", metavar='DIR') wallets = Strings( "Wallet files in 'wallet_dir' to load at startup.", ['default_wallet'] ) create_default_wallet = Toggle( "Create an initial wallet if it does not exist on startup.", True ) create_default_account = Toggle( "Create an initial account if it does not exist in the default wallet.", True ) # network use_upnp = Toggle( "Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward" "ports or have firewall rules you likely want to disable this.", True ) udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port']) tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port']) prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0) network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0') # routing table split_buckets_under_index = Integer( "Routing table bucket index below which we always split the bucket if given a new key to add to it and " "the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) " "will increase. This setting is used by seed nodes, you probably don't want to change it during normal " "use.", 1 ) # protocol timeouts download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0) blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0) peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0) node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT) # blob announcement and download save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True) blob_lru_cache_size = Integer( "LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when " "replying to a range request. Set to 0 to disable.", 32 ) announce_head_and_sd_only = Toggle( "Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True, previous_names=['announce_head_blobs_only'] ) concurrent_blob_announcers = Integer( "Number of blobs to iteratively announce at once, set to 0 to disable", 10, previous_names=['concurrent_announcers'] ) max_connections_per_download = Integer( "Maximum number of peers to connect to while downloading a blob", 4, previous_names=['max_connections_per_stream'] ) fixed_peer_delay = Float( "Amount of seconds before adding the reflector servers as potential peers to download from in case dht" "peers are not found or are slow", 2.0 ) max_key_fee = MaxKeyFee( "Don't download streams with fees exceeding this amount. When set to " "null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0} ) max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0") # reflector settings reflect_streams = Toggle( "Upload completed streams (published and downloaded) reflector in order to re-host them", True, previous_names=['reflect_uploads'] ) concurrent_reflector_uploads = Integer( "Maximum number of streams to upload to a reflector server at a time", 10 ) # servers reflector_servers = Servers("Reflector re-hosting servers", [ ('reflector.lbry.com', 5566) ]) lbryum_servers = Servers("SPV wallet servers", [ ('spv11.lbry.com', 50001), ('spv12.lbry.com', 50001), ('spv13.lbry.com', 50001), ('spv14.lbry.com', 50001), ('spv15.lbry.com', 50001), ('spv16.lbry.com', 50001), ('spv17.lbry.com', 50001), ('spv18.lbry.com', 50001), ('spv19.lbry.com', 50001), ]) known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [ ('lbrynet1.lbry.com', 4444), # US EAST ('lbrynet2.lbry.com', 4444), # US WEST ('lbrynet3.lbry.com', 4444), # EU ('lbrynet4.lbry.com', 4444) # ASIA ]) comment_server = String("Comment server API URL", "https://comments.lbry.com/api") # blockchain lbrycrd_dir = Path("Directory containing lbrycrd data.", metavar='DIR') blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main') spv_address_filters = Toggle( "Generate Golomb-Rice coding filters for blocks and transactions. Enables " "light client to synchronize with a full node.", True ) # daemon save_files = Toggle("Save downloaded files when calling `get` by default", True) components_to_skip = Strings("components which will be skipped during start-up of daemon", []) share_usage_data = Toggle( "Whether to share usage stats and diagnostic info with LBRY.", False, previous_names=['upload_log', 'upload_log', 'share_debug_info'] ) track_bandwidth = Toggle("Track bandwidth usage", True) # media server streaming_server = String('Host name and port to serve streaming media over range requests', 'localhost:5280', metavar='HOST:PORT') streaming_get = Toggle("Enable the /get endpoint for the streaming media server. " "Disable to prevent new streams from being added.", True) coin_selection_strategy = StringChoice( "Strategy to use when selecting UTXOs for a transaction", COIN_SELECTION_STRATEGIES, "standard") save_resolved_claims = Toggle( "Save content claims to the database when they are resolved to keep file_list up to date, " "only disable this if file_x commands are not needed", True ) @property def streaming_host(self): return self.streaming_server.split(':')[0] @property def streaming_port(self): return int(self.streaming_server.split(':')[1]) def __init__(self, **kwargs): super().__init__(**kwargs) self.set_default_paths() @classmethod def with_same_dir(cls, same_dir): return cls( data_dir=same_dir, download_dir=same_dir, wallet_dir=same_dir, lbrycrd_dir=same_dir, ) def set_default_paths(self): if 'darwin' in sys.platform.lower(): get_directories = get_darwin_directories elif 'win' in sys.platform.lower(): get_directories = get_windows_directories elif 'linux' in sys.platform.lower(): get_directories = get_linux_directories else: return cls = type(self) cls.data_dir.default, cls.wallet_dir.default,\ cls.blob_dir.default, cls.download_dir.default = get_directories() old_settings_file = os.path.join(self.data_dir, 'daemon_settings.yml') if os.path.exists(old_settings_file): cls.config.default = old_settings_file else: cls.config.default = os.path.join(self.data_dir, 'settings.yml') if self.data_dir != cls.data_dir.default: cls.blob_dir.default = os.path.join(self.data_dir, 'blobs') cls.wallet_dir.default = os.path.join(self.data_dir, 'wallets') @property def log_file_path(self): return os.path.join(self.data_dir, 'daemon.log') @property def db_url_or_default(self): if self.db_url: return self.db_url return 'sqlite:///'+os.path.join(self.data_dir, f'{self.blockchain_name}.db') def get_windows_directories() -> Tuple[str, str, str, str]: # very old data_dir = user_data_dir('lbrynet', roaming=True) blob_dir = os.path.join(data_dir, 'blobfiles') wallet_dir = os.path.join(user_data_dir('lbryum', roaming=True), 'wallets') if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir): return data_dir, wallet_dir, blob_dir, user_download_dir() # old data_dir = user_data_dir('lbrynet', 'lbry') blob_dir = os.path.join(data_dir, 'blobfiles') wallet_dir = os.path.join(user_data_dir('lbryum', 'lbry'), 'wallets') if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir): return data_dir, wallet_dir, blob_dir, user_download_dir() # new return get_universal_directories() def get_darwin_directories() -> Tuple[str, str, str, str]: data_dir = user_data_dir('LBRY') blob_dir = os.path.join(data_dir, 'blobfiles') wallet_dir = os.path.expanduser('~/.lbryum/wallets') if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir): return data_dir, wallet_dir, blob_dir, user_download_dir() return get_universal_directories() def get_linux_directories() -> Tuple[str, str, str, str]: # very old data_dir = os.path.expanduser('~/.lbrynet') blob_dir = os.path.join(data_dir, 'blobfiles') wallet_dir = os.path.join(os.path.expanduser('~/.lbryum'), 'wallets') if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir): return data_dir, wallet_dir, blob_dir, user_download_dir() # old data_dir = user_data_dir('lbry/lbrynet') blob_dir = os.path.join(data_dir, 'blobfiles') wallet_dir = user_data_dir('lbry/lbryum/wallets') if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir): return data_dir, wallet_dir, blob_dir, user_download_dir() # new return get_universal_directories() def get_universal_directories() -> Tuple[str, str, str, str]: lbrynet_dir = user_data_dir('lbrynet') return ( lbrynet_dir, os.path.join(lbrynet_dir, 'wallets'), os.path.join(lbrynet_dir, 'blobs'), user_download_dir() )