From 272e074a150fab16cffb415ec3be5d9d089bb697 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 14 Aug 2016 01:00:49 -0400 Subject: [PATCH 1/9] change content-type to content_type, make changing metadata easier --- lbrynet/core/LBRYMetadata.py | 389 ++++++++++++++++++------ lbrynet/lbrynet_daemon/LBRYPublisher.py | 6 +- tests/lbrynet/core/test_LBRYMetadata.py | 2 +- 3 files changed, 298 insertions(+), 99 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 5c101078b..0f6daa537 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -1,36 +1,13 @@ import json - +import logging from copy import deepcopy from lbrynet.conf import CURRENCIES -from lbrynet.core import utils -import logging +from distutils.version import StrictVersion log = logging.getLogger(__name__) -BITTREX_FEE = 0.0025 - -# Metadata version SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-' -BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] -OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] - -MV001 = "0.0.1" -MV002 = "0.0.2" -CURRENT_METADATA_VERSION = MV002 - -METADATA_REVISIONS = {} -METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS} -METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']} - -# Fee version -BASE_FEE_FIELDS = ['amount', 'address'] - -FV001 = "0.0.1" -CURRENT_FEE_REVISION = FV001 - -FEE_REVISIONS = {} -FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []} def verify_name_characters(name): @@ -39,18 +16,238 @@ def verify_name_characters(name): return True -class LBRYFeeValidator(dict): - def __init__(self, fee_dict): +def skip_validate(value): + pass + + +def verify_supported_currency(fee): + assert len(fee) == 1 + for c in fee: + assert c in CURRENCIES + + +def validate_sources(sources): + for source in sources: + assert source in SOURCE_TYPES, "Unknown source type" + + +class Validator(dict): + """ + Base class for validated dictionaries + """ + + DO_NOTHING = "pass" + UPDATE = "update_key" + IF_KEY = "if_key_exists" + REQUIRE = "require" + SKIP = "skip" + OPTIONAL = "add_optional" + ADD = "add" + IF_VAL = "if_val" + SUPERSEDE = "supersede" + + # override these + current_version = None + versions = None + migrations = None + supersessions = None + + @classmethod + def load_from_hex(cls, hex_val): + return cls(json.loads(hex_val.decode('hex'))) + + def process(self): + unprocessed = deepcopy(self) + if self.migrations is not None: + self._migrate_value(unprocessed) + + def serialize(self): + return json.dumps(self).encode("hex") + + def as_json(self): + return json.dumps(self) + + def __init__(self, value, process_now=True): dict.__init__(self) - assert len(fee_dict) == 1 - self.fee_version = None - self.currency_symbol = None + self._skip = [] + value_to_load = deepcopy(value) + if self.supersessions is not None: + self._run_supersessions(value_to_load) + self._verify_value(value_to_load) + self._raw = deepcopy(self) + self.version = self.get('ver', "0.0.1") + if process_now: + self.process() - fee_to_load = deepcopy(fee_dict) + def _handle(self, cmd_tpl, value): + if cmd_tpl == self.DO_NOTHING: + return - for currency in fee_dict: - self._verify_fee(currency, fee_to_load) + cmd = cmd_tpl[0] + if cmd == self.IF_KEY: + key, on_key, on_else = cmd_tpl[1:] + if key in value: + return self._handle(on_key, value) + elif on_else: + return self._handle(on_else, value) + return + + elif cmd == self.IF_VAL: + key, v, on_val, on_else = cmd_tpl[1:] + if key not in value: + return self._handle(on_else, value) + if value[key] == v: + return self._handle(on_val, value) + elif on_else: + return self._handle(on_else, value) + return + + elif cmd == self.UPDATE: + old_key, new_key = cmd_tpl[1:] + value.update({new_key: value.pop(old_key)}) + + elif cmd == self.REQUIRE: + required, validator = cmd_tpl[1:] + if required not in self._skip: + assert required in value if required not in self else True, "Missing required field: %s, %s" % (required, self.as_json()) + if required not in self._skip and required in value: + self.update({required: value.pop(required)}) + validator(self[required]) + else: + pass + + elif cmd == self.OPTIONAL: + optional, validator = cmd_tpl[1:] + if optional in value and optional not in self._skip: + self.update({optional: value.pop(optional)}) + validator(self[optional]) + else: + pass + + elif cmd == self.SKIP: + to_skip = cmd_tpl[1] + self._skip.append(to_skip) + + elif cmd == self.ADD: + key, pushed_val = cmd_tpl[1:] + self.update({key: pushed_val}) + + elif cmd == self.SUPERSEDE: + ver = cmd_tpl[1] + self.update({'ver': ver}) + self.version = ver + + def _load_revision(self, version, value): + for k in self.versions[version]: + self._handle(k, value) + + def _verify_value(self, value): + for version in sorted(self.versions, key=StrictVersion): + self._load_revision(version, value) + if not value: + self['ver'] = version + break + for skip in self._skip: + if skip in value: + value.pop(skip) + assert value == {}, "Unknown keys: %s, %s" % (json.dumps(value.keys()), self.as_json()) + + def _migrate_value(self, value): + for migration in self.migrations: + self._run_migration(migration, value) + + def _run_migration(self, commands, value): + for cmd in commands: + self._handle(cmd, value) + + def _run_supersessions(self, value): + for cmd in self.supersessions: + self._handle(cmd, value) + + +class LBCFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = None + + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class BTCFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = None + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class USDFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = None + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class LBRYFeeValidator(Validator): + CV001 = "0.0.1" + CURRENT_CURRENCY_VERSION = CV001 + + CURRENCY_REVISIONS = {} + + CURRENCY_REVISIONS[CV001] = [ + (Validator.OPTIONAL, 'BTC', BTCFeeValidator), + (Validator.OPTIONAL, 'USD', USDFeeValidator), + (Validator.OPTIONAL, 'LBC', LBCFeeValidator), + ] + + CURRENCY_MIGRATIONS = None + + current_version = CURRENT_CURRENCY_VERSION + versions = CURRENCY_REVISIONS + migrations = CURRENCY_MIGRATIONS + + def __init__(self, fee_dict): + Validator.__init__(self, fee_dict) + self.currency_symbol = self.keys()[0] self.amount = self._get_amount() self.address = self[self.currency_symbol]['address'] @@ -65,71 +262,73 @@ class LBRYFeeValidator(dict): log.error('Failed to convert %s to float', amt) raise - def _verify_fee(self, currency, f): - # str in case someone made a claim with a wierd fee - assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) - self.currency_symbol = currency - self.update({currency: {}}) - for version in FEE_REVISIONS: - self._load_revision(version, f) - if not f: - self.fee_version = version - break - assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) - def _load_revision(self, version, f): - for k in FEE_REVISIONS[version]['required']: - assert k in f[self.currency_symbol], "Missing required fee field: %s" % k - self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) - for k in FEE_REVISIONS[version]['optional']: - if k in f[self.currency_symbol]: - self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) +class Metadata(Validator): + MV001 = "0.0.1" + MV002 = "0.0.2" + MV003 = "0.0.3" + CURRENT_METADATA_VERSION = MV003 + METADATA_REVISIONS = {} -class Metadata(dict): - @classmethod - def load_from_hex(cls, metadata): - return cls(json.loads(metadata.decode('hex'))) + METADATA_REVISIONS[MV001] = [ + (Validator.REQUIRE, 'title', skip_validate), + (Validator.REQUIRE, 'description', skip_validate), + (Validator.REQUIRE, 'author', skip_validate), + (Validator.REQUIRE, 'language', skip_validate), + (Validator.REQUIRE, 'license', skip_validate), + (Validator.REQUIRE, 'content-type', skip_validate), + (Validator.REQUIRE, 'sources', validate_sources), + (Validator.OPTIONAL, 'thumbnail', skip_validate), + (Validator.OPTIONAL, 'preview', skip_validate), + (Validator.OPTIONAL, 'fee', verify_supported_currency), + (Validator.OPTIONAL, 'contact', skip_validate), + (Validator.OPTIONAL, 'pubkey', skip_validate), + ] + + METADATA_REVISIONS[MV002] = [ + (Validator.REQUIRE, 'nsfw', skip_validate), + (Validator.REQUIRE, 'ver', skip_validate), + (Validator.OPTIONAL, 'license_url', skip_validate), + ] + + METADATA_REVISIONS[MV003] = [ + (Validator.REQUIRE, 'content_type', skip_validate), + (Validator.SKIP, 'content-type'), + (Validator.OPTIONAL, 'sig', skip_validate), + (Validator.IF_KEY, 'sig', (Validator.REQUIRE, 'pubkey', skip_validate), Validator.DO_NOTHING), + (Validator.IF_KEY, 'pubkey', (Validator.REQUIRE, 'sig', skip_validate), Validator.DO_NOTHING), + ] + + MIGRATE_MV001_TO_MV002 = [ + (Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.ADD, 'nsfw', False)), + (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV002)), + ] + MIGRATE_MV002_TO_MV003 = [ + (Validator.IF_KEY, 'content_type', Validator.DO_NOTHING, (Validator.UPDATE, 'content-type', 'content_type')), + (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV003)), + (Validator.IF_VAL, 'ver', MV002, (Validator.SUPERSEDE, MV003), Validator.DO_NOTHING), + ] + + METADATA_MIGRATIONS = [ + MIGRATE_MV001_TO_MV002, + MIGRATE_MV002_TO_MV003, + ] + + SUPERSESSIONS = [ + (Validator.SKIP, 'content-type'), + ] + + current_version = CURRENT_METADATA_VERSION + versions = METADATA_REVISIONS + migrations = METADATA_MIGRATIONS + supersessions = SUPERSESSIONS def __init__(self, metadata): - dict.__init__(self) - self.meta_version = None - metadata_to_load = deepcopy(metadata) + Validator.__init__(self, metadata) + self.meta_version = self.get('ver', Metadata.MV001) + self._load_fee() - self._verify_sources(metadata_to_load) - self._verify_metadata(metadata_to_load) - - def _load_revision(self, version, metadata): - for k in METADATA_REVISIONS[version]['required']: - assert k in metadata, "Missing required metadata field: %s" % k - self.update({k: metadata.pop(k)}) - for k in METADATA_REVISIONS[version]['optional']: - if k == 'fee': - self._load_fee(metadata) - elif k in metadata: - self.update({k: metadata.pop(k)}) - - def _load_fee(self, metadata): - if 'fee' in metadata: - self['fee'] = LBRYFeeValidator(metadata.pop('fee')) - - def _verify_sources(self, metadata): - assert "sources" in metadata, "No sources given" - for source in metadata['sources']: - assert source in SOURCE_TYPES, "Unknown source type" - - def _verify_metadata(self, metadata): - for version in METADATA_REVISIONS: - self._load_revision(version, metadata) - if not metadata: - self.meta_version = version - if utils.version_is_greater_than(self.meta_version, "0.0.1"): - assert self.meta_version == self['ver'], "version mismatch" - break - assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys()) - - def serialize(self): - return json.dumps(self).encode("hex") - - def as_json(self): - return json.dumps(self) + def _load_fee(self): + if 'fee' in self: + self.update({'fee': LBRYFeeValidator(self['fee'])}) \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 75ceb6093..de2dca070 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -9,7 +9,7 @@ from lbrynet.core.Error import InsufficientFundsError from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.core.PaymentRateManager import PaymentRateManager -from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION +from lbrynet.core.LBRYMetadata import Metadata from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader from lbrynet.conf import LOG_FILE_NAME from twisted.internet import threads, defer @@ -101,9 +101,9 @@ class Publisher(object): return d def _claim_name(self): - self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, + self.metadata['content_type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name))[0] - self.metadata['ver'] = CURRENT_METADATA_VERSION + self.metadata['ver'] = Metadata.current_version m = Metadata(self.metadata) def set_tx_hash(txid): diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index e5c3255b3..5a35c6507 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -21,7 +21,7 @@ class MetadataTest(unittest.TestCase): 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' } m = LBRYMetadata.Metadata(metadata) - self.assertFalse('key' in m) + self.assertFalse('fee' in m) def test_assertion_if_invalid_source(self): metadata = { From 1ca9f575efb4ec2a2490634b8d4d38a1075c446a Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 15 Aug 2016 17:07:50 -0400 Subject: [PATCH 2/9] decorator prettiness --- lbrynet/core/LBRYMetadata.py | 241 +++++++++++++++++++---------------- 1 file changed, 130 insertions(+), 111 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 0f6daa537..4651d9653 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -3,6 +3,7 @@ import logging from copy import deepcopy from lbrynet.conf import CURRENCIES from distutils.version import StrictVersion +from lbrynet.core.utils import version_is_greater_than log = logging.getLogger(__name__) @@ -17,49 +18,74 @@ def verify_name_characters(name): def skip_validate(value): - pass + return True def verify_supported_currency(fee): assert len(fee) == 1 for c in fee: assert c in CURRENCIES + return True def validate_sources(sources): for source in sources: - assert source in SOURCE_TYPES, "Unknown source type" + assert source in SOURCE_TYPES, "Unknown source type: %s" % str(source) + return True +def verify_amount(x): + return isinstance(x, float) and x > 0 + + +def processor(cls): + for methodname in dir(cls): + method = getattr(cls, methodname) + if hasattr(method, 'cmd_name'): + cls.commands.update({method.cmd_name: methodname}) + return cls + + +def cmd(cmd_name): + def wrapper(func): + func.cmd_name = cmd_name + return func + return wrapper + + +@processor class Validator(dict): """ Base class for validated dictionaries """ - DO_NOTHING = "pass" - UPDATE = "update_key" - IF_KEY = "if_key_exists" - REQUIRE = "require" - SKIP = "skip" - OPTIONAL = "add_optional" - ADD = "add" - IF_VAL = "if_val" - SUPERSEDE = "supersede" - # override these current_version = None versions = None migrations = None - supersessions = None + + # built in commands + DO_NOTHING = "do_nothing" + UPDATE = "update_key" + IF_KEY = "if_key" + REQUIRE = "require" + SKIP = "skip" + OPTIONAL = "optional" + LOAD = "load" + IF_VAL = "if_val" + + commands = {} @classmethod def load_from_hex(cls, hex_val): return cls(json.loads(hex_val.decode('hex'))) - def process(self): - unprocessed = deepcopy(self) - if self.migrations is not None: - self._migrate_value(unprocessed) + @classmethod + def validate(cls, value): + if cls(value): + return True + else: + return False def serialize(self): return json.dumps(self).encode("hex") @@ -67,91 +93,90 @@ class Validator(dict): def as_json(self): return json.dumps(self) - def __init__(self, value, process_now=True): + def __init__(self, value, process_now=False): dict.__init__(self) self._skip = [] value_to_load = deepcopy(value) - if self.supersessions is not None: - self._run_supersessions(value_to_load) - self._verify_value(value_to_load) - self._raw = deepcopy(self) - self.version = self.get('ver', "0.0.1") if process_now: - self.process() + self.process(value_to_load) + self._verify_value(value_to_load) + self.version = self.get('ver', "0.0.1") + + def process(self, value): + if self.migrations is not None: + self._migrate_value(value) + + @cmd(DO_NOTHING) + def _do_nothing(self): + pass + + @cmd(SKIP) + def _add_to_skipped(self, rx_value, key): + if key not in self._skip: + self._skip.append(key) + + @cmd(UPDATE) + def _update(self, rx_value, old_key, new_key): + rx_value.update({new_key: rx_value.pop(old_key)}) + + @cmd(IF_KEY) + def _if_key(self, rx_value, key, if_true, if_else): + if key in rx_value: + self._handle(if_true, rx_value) + self._handle(if_else, rx_value) + + @cmd(IF_VAL) + def _if_val(self, rx_value, key, val, if_true, if_else): + if key in rx_value: + if rx_value[key] == val: + self._handle(if_true, rx_value) + self._handle(if_else, rx_value) + + @cmd(LOAD) + def _load(self, rx_value, key, value): + rx_value.update({key: value}) + + @cmd(REQUIRE) + def _require(self, rx_value, key, validator=None): + if key not in self._skip: + assert key in rx_value, "Key is missing: %s" % key + if isinstance(validator, type): + assert isinstance(rx_value[key], validator), "%s: %s isn't required %s" % (key, type(rx_value[key]), validator) + elif callable(validator): + assert validator(rx_value[key]), "Failed to validate %s" % key + self.update({key: rx_value.pop(key)}) + + @cmd(OPTIONAL) + def _optional(self, rx_value, key, validator=None): + if key in rx_value and key not in self._skip: + if isinstance(validator, type): + assert isinstance(rx_value[key], validator), "%s type %s isn't required %s" % (key, type(rx_value[key]), validator) + elif callable(validator): + assert validator(rx_value[key]), "Failed to validate %s" % key + self.update({key: rx_value.pop(key)}) def _handle(self, cmd_tpl, value): - if cmd_tpl == self.DO_NOTHING: + if cmd_tpl == Validator.DO_NOTHING: return - - cmd = cmd_tpl[0] - - if cmd == self.IF_KEY: - key, on_key, on_else = cmd_tpl[1:] - if key in value: - return self._handle(on_key, value) - elif on_else: - return self._handle(on_else, value) - return - - elif cmd == self.IF_VAL: - key, v, on_val, on_else = cmd_tpl[1:] - if key not in value: - return self._handle(on_else, value) - if value[key] == v: - return self._handle(on_val, value) - elif on_else: - return self._handle(on_else, value) - return - - elif cmd == self.UPDATE: - old_key, new_key = cmd_tpl[1:] - value.update({new_key: value.pop(old_key)}) - - elif cmd == self.REQUIRE: - required, validator = cmd_tpl[1:] - if required not in self._skip: - assert required in value if required not in self else True, "Missing required field: %s, %s" % (required, self.as_json()) - if required not in self._skip and required in value: - self.update({required: value.pop(required)}) - validator(self[required]) - else: - pass - - elif cmd == self.OPTIONAL: - optional, validator = cmd_tpl[1:] - if optional in value and optional not in self._skip: - self.update({optional: value.pop(optional)}) - validator(self[optional]) - else: - pass - - elif cmd == self.SKIP: - to_skip = cmd_tpl[1] - self._skip.append(to_skip) - - elif cmd == self.ADD: - key, pushed_val = cmd_tpl[1:] - self.update({key: pushed_val}) - - elif cmd == self.SUPERSEDE: - ver = cmd_tpl[1] - self.update({'ver': ver}) - self.version = ver + command = cmd_tpl[0] + f = getattr(self, self.commands[command]) + if len(cmd_tpl) > 1: + args = (value,) + cmd_tpl[1:] + f(*args) + else: + f() def _load_revision(self, version, value): for k in self.versions[version]: self._handle(k, value) def _verify_value(self, value): - for version in sorted(self.versions, key=StrictVersion): + val_ver = value.get('ver', "0.0.1") + # verify version requirements in reverse order starting from the version asserted in the value + versions = sorted([v for v in self.versions if not version_is_greater_than(v, val_ver)], key=StrictVersion, reverse=True) + for version in versions: self._load_revision(version, value) - if not value: - self['ver'] = version - break - for skip in self._skip: - if skip in value: - value.pop(skip) - assert value == {}, "Unknown keys: %s, %s" % (json.dumps(value.keys()), self.as_json()) + assert value == {} or value.keys() == self._skip, "Unknown keys: %s" % json.dumps(value) def _migrate_value(self, value): for migration in self.migrations: @@ -161,10 +186,6 @@ class Validator(dict): for cmd in commands: self._handle(cmd, value) - def _run_supersessions(self, value): - for cmd in self.supersessions: - self._handle(cmd, value) - class LBCFeeValidator(Validator): FV001 = "0.0.1" @@ -173,7 +194,7 @@ class LBCFeeValidator(Validator): FEE_REVISIONS = {} FEE_REVISIONS[FV001] = [ - (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'amount', verify_amount), (Validator.REQUIRE, 'address', skip_validate), ] @@ -194,11 +215,12 @@ class BTCFeeValidator(Validator): FEE_REVISIONS = {} FEE_REVISIONS[FV001] = [ - (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'amount',verify_amount), (Validator.REQUIRE, 'address', skip_validate), ] FEE_MIGRATIONS = None + current_version = CURRENT_FEE_VERSION versions = FEE_REVISIONS migrations = FEE_MIGRATIONS @@ -214,11 +236,12 @@ class USDFeeValidator(Validator): FEE_REVISIONS = {} FEE_REVISIONS[FV001] = [ - (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'amount',verify_amount), (Validator.REQUIRE, 'address', skip_validate), ] FEE_MIGRATIONS = None + current_version = CURRENT_FEE_VERSION versions = FEE_REVISIONS migrations = FEE_MIGRATIONS @@ -234,9 +257,9 @@ class LBRYFeeValidator(Validator): CURRENCY_REVISIONS = {} CURRENCY_REVISIONS[CV001] = [ - (Validator.OPTIONAL, 'BTC', BTCFeeValidator), - (Validator.OPTIONAL, 'USD', USDFeeValidator), - (Validator.OPTIONAL, 'LBC', LBCFeeValidator), + (Validator.OPTIONAL, 'BTC', BTCFeeValidator.validate), + (Validator.OPTIONAL, 'USD', USDFeeValidator.validate), + (Validator.OPTIONAL, 'LBC', LBCFeeValidator.validate), ] CURRENCY_MIGRATIONS = None @@ -301,13 +324,13 @@ class Metadata(Validator): ] MIGRATE_MV001_TO_MV002 = [ - (Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.ADD, 'nsfw', False)), - (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV002)), + (Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.LOAD, 'nsfw', False)), + (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.LOAD, 'ver', MV002)), ] + MIGRATE_MV002_TO_MV003 = [ - (Validator.IF_KEY, 'content_type', Validator.DO_NOTHING, (Validator.UPDATE, 'content-type', 'content_type')), - (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV003)), - (Validator.IF_VAL, 'ver', MV002, (Validator.SUPERSEDE, MV003), Validator.DO_NOTHING), + (Validator.IF_VAL, 'ver', MV002, (Validator.UPDATE, 'content-type', 'content_type'), Validator.DO_NOTHING), + (Validator.IF_VAL, 'ver', MV002, (Validator.LOAD, 'ver', MV003), Validator.DO_NOTHING), ] METADATA_MIGRATIONS = [ @@ -315,20 +338,16 @@ class Metadata(Validator): MIGRATE_MV002_TO_MV003, ] - SUPERSESSIONS = [ - (Validator.SKIP, 'content-type'), - ] - current_version = CURRENT_METADATA_VERSION versions = METADATA_REVISIONS migrations = METADATA_MIGRATIONS - supersessions = SUPERSESSIONS - def __init__(self, metadata): - Validator.__init__(self, metadata) + def __init__(self, metadata, process_now=True): + Validator.__init__(self, metadata, process_now) self.meta_version = self.get('ver', Metadata.MV001) self._load_fee() def _load_fee(self): if 'fee' in self: - self.update({'fee': LBRYFeeValidator(self['fee'])}) \ No newline at end of file + self.update({'fee': LBRYFeeValidator(self['fee'])}) + From 66396de6a35e071ea9469c952ecc33726fb3f613 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 15 Aug 2016 17:41:38 -0400 Subject: [PATCH 3/9] remove as_json and serialize --- lbrynet/core/LBRYMetadata.py | 6 ------ lbrynet/core/LBRYWallet.py | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 4651d9653..ca6a063f1 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -87,12 +87,6 @@ class Validator(dict): else: return False - def serialize(self): - return json.dumps(self).encode("hex") - - def as_json(self): - return json.dumps(self) - def __init__(self, value, process_now=False): dict.__init__(self) self._skip = [] diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 0bc584aca..26a8c7b4a 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -1218,7 +1218,7 @@ class LBRYumWallet(LBRYWallet): def _send_name_claim_update(self, name, claim_id, txid, value, amount): def send_claim_update(address): decoded_claim_id = claim_id.decode('hex')[::-1] - metadata = Metadata(value).as_json() + metadata = json.dumps(Metadata(value)) log.info("updateclaim %s %s %f %s %s '%s'", txid, address, amount, name, decoded_claim_id.encode('hex'), json.dumps(metadata)) cmd = known_commands['updateclaim'] func = getattr(self.cmd_runner, cmd.name) From d8c4e2a72ed3787e06abe26eebf005f2d6bbc264 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 15 Aug 2016 17:54:43 -0400 Subject: [PATCH 4/9] whoops --- lbrynet/core/LBRYMetadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index ca6a063f1..33b5e3a5c 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -116,15 +116,15 @@ class Validator(dict): @cmd(IF_KEY) def _if_key(self, rx_value, key, if_true, if_else): if key in rx_value: - self._handle(if_true, rx_value) - self._handle(if_else, rx_value) + return self._handle(if_true, rx_value) + return self._handle(if_else, rx_value) @cmd(IF_VAL) def _if_val(self, rx_value, key, val, if_true, if_else): if key in rx_value: if rx_value[key] == val: - self._handle(if_true, rx_value) - self._handle(if_else, rx_value) + return self._handle(if_true, rx_value) + return self._handle(if_else, rx_value) @cmd(LOAD) def _load(self, rx_value, key, value): From 072ff4b02ed34a5af4bc7d8a41edb4bd49d1fa00 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 15 Aug 2016 18:13:36 -0400 Subject: [PATCH 5/9] fix tests --- lbrynet/core/LBRYWallet.py | 4 +- tests/lbrynet/core/test_LBRYMetadata.py | 102 +++++++++++++++++++----- 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 26a8c7b4a..b71b4bc22 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -322,7 +322,7 @@ class LBRYWallet(object): assert k in r, "getvalueforname response missing field %s" % k def _log_success(claim_id): - log.info("lbry://%s complies with %s, claimid: %s", name, metadata.meta_version, claim_id) + log.info("lbry://%s complies with %s, claimid: %s", name, metadata.version, claim_id) return defer.succeed(None) if 'error' in result: @@ -377,7 +377,7 @@ class LBRYWallet(object): result = {} try: metadata = Metadata(json.loads(claim['value'])) - meta_ver = metadata.meta_version + meta_ver = metadata.version sd_hash = metadata['sources']['lbry_sd_hash'] d = self._save_name_metadata(name, txid, sd_hash) except AssertionError: diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index 5a35c6507..4a64cfea6 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -18,7 +18,7 @@ class MetadataTest(unittest.TestCase): 'sources': { 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, 'content-type': 'audio/mpeg', - 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg', } m = LBRYMetadata.Metadata(metadata) self.assertFalse('fee' in m) @@ -26,7 +26,6 @@ class MetadataTest(unittest.TestCase): def test_assertion_if_invalid_source(self): metadata = { 'license': 'Oscilloscope Laboratories', - 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', 'language': 'en', 'title': "It's a Disaster", @@ -34,7 +33,7 @@ class MetadataTest(unittest.TestCase): 'sources': { 'fake': 'source'}, 'content-type': 'audio/mpeg', - 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg', } with self.assertRaises(AssertionError): LBRYMetadata.Metadata(metadata) @@ -57,7 +56,6 @@ class MetadataTest(unittest.TestCase): def test_version_is_001_if_all_fields_are_present(self): metadata = { 'license': 'Oscilloscope Laboratories', - 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', 'language': 'en', 'title': "It's a Disaster", @@ -65,27 +63,25 @@ class MetadataTest(unittest.TestCase): 'sources': { 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, 'content-type': 'audio/mpeg', - 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg', } - m = LBRYMetadata.Metadata(metadata) - self.assertEquals('0.0.1', m.meta_version) + m = LBRYMetadata.Metadata(metadata, process_now=False) + self.assertEquals('0.0.1', m.version) def test_assertion_if_there_is_an_extra_field(self): metadata = { - 'license': 'NASA', - 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, - 'ver': '0.0.2', - 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'license': 'Oscilloscope Laboratories', + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', 'language': 'en', - 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', - 'title': 'Thermonuclear Art', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', 'sources': { - 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, - 'nsfw': False, - 'content-type': 'video/mp4', - 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg', + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg', 'MYSTERYFIELD': '?' } + m = LBRYMetadata.Metadata(metadata, process_now=False) with self.assertRaises(AssertionError): LBRYMetadata.Metadata(metadata) @@ -104,8 +100,26 @@ class MetadataTest(unittest.TestCase): 'content-type': 'video/mp4', 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' } - m = LBRYMetadata.Metadata(metadata) - self.assertEquals('0.0.2', m.meta_version) + m = LBRYMetadata.Metadata(metadata, process_now=False) + self.assertEquals('0.0.2', m.version) + + def test_version_is_003_if_all_fields_are_present(self): + metadata = { + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.3', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content_type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' + } + m = LBRYMetadata.Metadata(metadata, process_now=False) + self.assertEquals('0.0.3', m.version) def test_version_claimed_is_001_but_version_is_002(self): metadata = { @@ -123,7 +137,55 @@ class MetadataTest(unittest.TestCase): 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' } with self.assertRaises(AssertionError): - LBRYMetadata.Metadata(metadata) + LBRYMetadata.Metadata(metadata, process_now=False) + def test_version_claimed_is_002_but_version_is_003(self): + metadata = { + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.2', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content_type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata, process_now=False) + def test_version_001_ports_to_003(self): + metadata = { + 'license': 'Oscilloscope Laboratories', + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg', + } + m = LBRYMetadata.Metadata(metadata, process_now=True) + self.assertEquals('0.0.3', m.version) + def test_version_002_ports_to_003(self): + metadata = { + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.2', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content-type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' + } + m = LBRYMetadata.Metadata(metadata, process_now=True) + self.assertEquals('0.0.3', m.version) \ No newline at end of file From 75d1fa585c80a94aae62c6a5d31a9a4d4dffc703 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 15 Aug 2016 18:30:14 -0400 Subject: [PATCH 6/9] fix tests --- lbrynet/core/LBRYMetadata.py | 2 +- tests/lbrynet/core/test_LBRYMetadata.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 33b5e3a5c..a4c2b4fbc 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -323,7 +323,7 @@ class Metadata(Validator): ] MIGRATE_MV002_TO_MV003 = [ - (Validator.IF_VAL, 'ver', MV002, (Validator.UPDATE, 'content-type', 'content_type'), Validator.DO_NOTHING), + (Validator.IF_KEY, 'content-type', (Validator.UPDATE, 'content-type', 'content_type'), Validator.DO_NOTHING), (Validator.IF_VAL, 'ver', MV002, (Validator.LOAD, 'ver', MV003), Validator.DO_NOTHING), ] diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index 4a64cfea6..953b8221f 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -3,11 +3,24 @@ from twisted.trial import unittest class MetadataTest(unittest.TestCase): - def test_assertion_if_source_is_missing(self): + def test_assertion_if_no_metadata(self): metadata = {} with self.assertRaises(AssertionError): LBRYMetadata.Metadata(metadata) + def test_assertion_if_source_is_missing(self): + metadata = { + 'license': 'Oscilloscope Laboratories', + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg', + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + def test_metadata_works_without_fee(self): metadata = { 'license': 'Oscilloscope Laboratories', @@ -81,9 +94,8 @@ class MetadataTest(unittest.TestCase): 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg', 'MYSTERYFIELD': '?' } - m = LBRYMetadata.Metadata(metadata, process_now=False) with self.assertRaises(AssertionError): - LBRYMetadata.Metadata(metadata) + LBRYMetadata.Metadata(metadata, process_now=False) def test_version_is_002_if_all_fields_are_present(self): metadata = { From e8fe1cd14de7fe69d03cdd7f60ec91a595c347ea Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 15 Aug 2016 18:39:09 -0400 Subject: [PATCH 7/9] more test fixes --- lbrynet/core/LBRYMetadata.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index a4c2b4fbc..ef2c4d6a4 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -61,8 +61,8 @@ class Validator(dict): # override these current_version = None - versions = None - migrations = None + versions = {} + migrations = [] # built in commands DO_NOTHING = "do_nothing" @@ -192,7 +192,7 @@ class LBCFeeValidator(Validator): (Validator.REQUIRE, 'address', skip_validate), ] - FEE_MIGRATIONS = None + FEE_MIGRATIONS = [] current_version = CURRENT_FEE_VERSION versions = FEE_REVISIONS @@ -213,7 +213,7 @@ class BTCFeeValidator(Validator): (Validator.REQUIRE, 'address', skip_validate), ] - FEE_MIGRATIONS = None + FEE_MIGRATIONS = [] current_version = CURRENT_FEE_VERSION versions = FEE_REVISIONS @@ -234,7 +234,7 @@ class USDFeeValidator(Validator): (Validator.REQUIRE, 'address', skip_validate), ] - FEE_MIGRATIONS = None + FEE_MIGRATIONS = [] current_version = CURRENT_FEE_VERSION versions = FEE_REVISIONS @@ -256,7 +256,7 @@ class LBRYFeeValidator(Validator): (Validator.OPTIONAL, 'LBC', LBCFeeValidator.validate), ] - CURRENCY_MIGRATIONS = None + CURRENCY_MIGRATIONS = [] current_version = CURRENT_CURRENCY_VERSION versions = CURRENCY_REVISIONS From 0f7ca8f24e84ee580d76f8dcf7e0470f9ca364a4 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 17 Aug 2016 12:28:15 -0400 Subject: [PATCH 8/9] move metadata and validator classes into their own directory --- lbrynet/core/LBRYMetadata.py | 347 ------------------ lbrynet/core/LBRYWallet.py | 2 +- lbrynet/lbrynet_daemon/LBRYDaemon.py | 3 +- lbrynet/lbrynet_daemon/LBRYDownloader.py | 2 +- .../lbrynet_daemon/LBRYExchangeRateManager.py | 2 +- lbrynet/lbrynet_daemon/LBRYPublisher.py | 2 +- lbrynet/metadata/LBRYFee.py | 116 ++++++ lbrynet/metadata/LBRYMetadata.py | 86 +++++ lbrynet/metadata/Validator.py | 155 ++++++++ lbrynet/metadata/__init__.py | 0 10 files changed, 362 insertions(+), 353 deletions(-) delete mode 100644 lbrynet/core/LBRYMetadata.py create mode 100644 lbrynet/metadata/LBRYFee.py create mode 100644 lbrynet/metadata/LBRYMetadata.py create mode 100644 lbrynet/metadata/Validator.py create mode 100644 lbrynet/metadata/__init__.py diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py deleted file mode 100644 index ef2c4d6a4..000000000 --- a/lbrynet/core/LBRYMetadata.py +++ /dev/null @@ -1,347 +0,0 @@ -import json -import logging -from copy import deepcopy -from lbrynet.conf import CURRENCIES -from distutils.version import StrictVersion -from lbrynet.core.utils import version_is_greater_than - -log = logging.getLogger(__name__) - -SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] -NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-' - - -def verify_name_characters(name): - for c in name: - assert c in NAME_ALLOWED_CHARSET, "Invalid character" - return True - - -def skip_validate(value): - return True - - -def verify_supported_currency(fee): - assert len(fee) == 1 - for c in fee: - assert c in CURRENCIES - return True - - -def validate_sources(sources): - for source in sources: - assert source in SOURCE_TYPES, "Unknown source type: %s" % str(source) - return True - - -def verify_amount(x): - return isinstance(x, float) and x > 0 - - -def processor(cls): - for methodname in dir(cls): - method = getattr(cls, methodname) - if hasattr(method, 'cmd_name'): - cls.commands.update({method.cmd_name: methodname}) - return cls - - -def cmd(cmd_name): - def wrapper(func): - func.cmd_name = cmd_name - return func - return wrapper - - -@processor -class Validator(dict): - """ - Base class for validated dictionaries - """ - - # override these - current_version = None - versions = {} - migrations = [] - - # built in commands - DO_NOTHING = "do_nothing" - UPDATE = "update_key" - IF_KEY = "if_key" - REQUIRE = "require" - SKIP = "skip" - OPTIONAL = "optional" - LOAD = "load" - IF_VAL = "if_val" - - commands = {} - - @classmethod - def load_from_hex(cls, hex_val): - return cls(json.loads(hex_val.decode('hex'))) - - @classmethod - def validate(cls, value): - if cls(value): - return True - else: - return False - - def __init__(self, value, process_now=False): - dict.__init__(self) - self._skip = [] - value_to_load = deepcopy(value) - if process_now: - self.process(value_to_load) - self._verify_value(value_to_load) - self.version = self.get('ver', "0.0.1") - - def process(self, value): - if self.migrations is not None: - self._migrate_value(value) - - @cmd(DO_NOTHING) - def _do_nothing(self): - pass - - @cmd(SKIP) - def _add_to_skipped(self, rx_value, key): - if key not in self._skip: - self._skip.append(key) - - @cmd(UPDATE) - def _update(self, rx_value, old_key, new_key): - rx_value.update({new_key: rx_value.pop(old_key)}) - - @cmd(IF_KEY) - def _if_key(self, rx_value, key, if_true, if_else): - if key in rx_value: - return self._handle(if_true, rx_value) - return self._handle(if_else, rx_value) - - @cmd(IF_VAL) - def _if_val(self, rx_value, key, val, if_true, if_else): - if key in rx_value: - if rx_value[key] == val: - return self._handle(if_true, rx_value) - return self._handle(if_else, rx_value) - - @cmd(LOAD) - def _load(self, rx_value, key, value): - rx_value.update({key: value}) - - @cmd(REQUIRE) - def _require(self, rx_value, key, validator=None): - if key not in self._skip: - assert key in rx_value, "Key is missing: %s" % key - if isinstance(validator, type): - assert isinstance(rx_value[key], validator), "%s: %s isn't required %s" % (key, type(rx_value[key]), validator) - elif callable(validator): - assert validator(rx_value[key]), "Failed to validate %s" % key - self.update({key: rx_value.pop(key)}) - - @cmd(OPTIONAL) - def _optional(self, rx_value, key, validator=None): - if key in rx_value and key not in self._skip: - if isinstance(validator, type): - assert isinstance(rx_value[key], validator), "%s type %s isn't required %s" % (key, type(rx_value[key]), validator) - elif callable(validator): - assert validator(rx_value[key]), "Failed to validate %s" % key - self.update({key: rx_value.pop(key)}) - - def _handle(self, cmd_tpl, value): - if cmd_tpl == Validator.DO_NOTHING: - return - command = cmd_tpl[0] - f = getattr(self, self.commands[command]) - if len(cmd_tpl) > 1: - args = (value,) + cmd_tpl[1:] - f(*args) - else: - f() - - def _load_revision(self, version, value): - for k in self.versions[version]: - self._handle(k, value) - - def _verify_value(self, value): - val_ver = value.get('ver', "0.0.1") - # verify version requirements in reverse order starting from the version asserted in the value - versions = sorted([v for v in self.versions if not version_is_greater_than(v, val_ver)], key=StrictVersion, reverse=True) - for version in versions: - self._load_revision(version, value) - assert value == {} or value.keys() == self._skip, "Unknown keys: %s" % json.dumps(value) - - def _migrate_value(self, value): - for migration in self.migrations: - self._run_migration(migration, value) - - def _run_migration(self, commands, value): - for cmd in commands: - self._handle(cmd, value) - - -class LBCFeeValidator(Validator): - FV001 = "0.0.1" - CURRENT_FEE_VERSION = FV001 - - FEE_REVISIONS = {} - - FEE_REVISIONS[FV001] = [ - (Validator.REQUIRE, 'amount', verify_amount), - (Validator.REQUIRE, 'address', skip_validate), - ] - - FEE_MIGRATIONS = [] - - current_version = CURRENT_FEE_VERSION - versions = FEE_REVISIONS - migrations = FEE_MIGRATIONS - - def __init__(self, fee): - Validator.__init__(self, fee) - - -class BTCFeeValidator(Validator): - FV001 = "0.0.1" - CURRENT_FEE_VERSION = FV001 - - FEE_REVISIONS = {} - - FEE_REVISIONS[FV001] = [ - (Validator.REQUIRE, 'amount',verify_amount), - (Validator.REQUIRE, 'address', skip_validate), - ] - - FEE_MIGRATIONS = [] - - current_version = CURRENT_FEE_VERSION - versions = FEE_REVISIONS - migrations = FEE_MIGRATIONS - - def __init__(self, fee): - Validator.__init__(self, fee) - - -class USDFeeValidator(Validator): - FV001 = "0.0.1" - CURRENT_FEE_VERSION = FV001 - - FEE_REVISIONS = {} - - FEE_REVISIONS[FV001] = [ - (Validator.REQUIRE, 'amount',verify_amount), - (Validator.REQUIRE, 'address', skip_validate), - ] - - FEE_MIGRATIONS = [] - - current_version = CURRENT_FEE_VERSION - versions = FEE_REVISIONS - migrations = FEE_MIGRATIONS - - def __init__(self, fee): - Validator.__init__(self, fee) - - -class LBRYFeeValidator(Validator): - CV001 = "0.0.1" - CURRENT_CURRENCY_VERSION = CV001 - - CURRENCY_REVISIONS = {} - - CURRENCY_REVISIONS[CV001] = [ - (Validator.OPTIONAL, 'BTC', BTCFeeValidator.validate), - (Validator.OPTIONAL, 'USD', USDFeeValidator.validate), - (Validator.OPTIONAL, 'LBC', LBCFeeValidator.validate), - ] - - CURRENCY_MIGRATIONS = [] - - current_version = CURRENT_CURRENCY_VERSION - versions = CURRENCY_REVISIONS - migrations = CURRENCY_MIGRATIONS - - def __init__(self, fee_dict): - Validator.__init__(self, fee_dict) - self.currency_symbol = self.keys()[0] - self.amount = self._get_amount() - self.address = self[self.currency_symbol]['address'] - - def _get_amount(self): - amt = self[self.currency_symbol]['amount'] - if isinstance(amt, float): - return amt - else: - try: - return float(amt) - except TypeError: - log.error('Failed to convert %s to float', amt) - raise - - -class Metadata(Validator): - MV001 = "0.0.1" - MV002 = "0.0.2" - MV003 = "0.0.3" - CURRENT_METADATA_VERSION = MV003 - - METADATA_REVISIONS = {} - - METADATA_REVISIONS[MV001] = [ - (Validator.REQUIRE, 'title', skip_validate), - (Validator.REQUIRE, 'description', skip_validate), - (Validator.REQUIRE, 'author', skip_validate), - (Validator.REQUIRE, 'language', skip_validate), - (Validator.REQUIRE, 'license', skip_validate), - (Validator.REQUIRE, 'content-type', skip_validate), - (Validator.REQUIRE, 'sources', validate_sources), - (Validator.OPTIONAL, 'thumbnail', skip_validate), - (Validator.OPTIONAL, 'preview', skip_validate), - (Validator.OPTIONAL, 'fee', verify_supported_currency), - (Validator.OPTIONAL, 'contact', skip_validate), - (Validator.OPTIONAL, 'pubkey', skip_validate), - ] - - METADATA_REVISIONS[MV002] = [ - (Validator.REQUIRE, 'nsfw', skip_validate), - (Validator.REQUIRE, 'ver', skip_validate), - (Validator.OPTIONAL, 'license_url', skip_validate), - ] - - METADATA_REVISIONS[MV003] = [ - (Validator.REQUIRE, 'content_type', skip_validate), - (Validator.SKIP, 'content-type'), - (Validator.OPTIONAL, 'sig', skip_validate), - (Validator.IF_KEY, 'sig', (Validator.REQUIRE, 'pubkey', skip_validate), Validator.DO_NOTHING), - (Validator.IF_KEY, 'pubkey', (Validator.REQUIRE, 'sig', skip_validate), Validator.DO_NOTHING), - ] - - MIGRATE_MV001_TO_MV002 = [ - (Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.LOAD, 'nsfw', False)), - (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.LOAD, 'ver', MV002)), - ] - - MIGRATE_MV002_TO_MV003 = [ - (Validator.IF_KEY, 'content-type', (Validator.UPDATE, 'content-type', 'content_type'), Validator.DO_NOTHING), - (Validator.IF_VAL, 'ver', MV002, (Validator.LOAD, 'ver', MV003), Validator.DO_NOTHING), - ] - - METADATA_MIGRATIONS = [ - MIGRATE_MV001_TO_MV002, - MIGRATE_MV002_TO_MV003, - ] - - current_version = CURRENT_METADATA_VERSION - versions = METADATA_REVISIONS - migrations = METADATA_MIGRATIONS - - def __init__(self, metadata, process_now=True): - Validator.__init__(self, metadata, process_now) - self.meta_version = self.get('ver', Metadata.MV001) - self._load_fee() - - def _load_fee(self): - if 'fee' in self: - self.update({'fee': LBRYFeeValidator(self['fee'])}) - diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index b71b4bc22..42909bce0 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -25,7 +25,7 @@ from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHand from lbrynet.core.client.ClientRequest import ClientRequest from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError from lbrynet.core.Error import InsufficientFundsError -from lbrynet.core.LBRYMetadata import Metadata +from lbrynet.metadata.LBRYMetadata import Metadata log = logging.getLogger(__name__) alert = logging.getLogger("lbryalert." + __name__) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 88313b2af..b0e477c01 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -40,10 +40,9 @@ from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient -from lbrynet.core.LBRYMetadata import Metadata +from lbrynet.metadata.LBRYMetadata import Metadata, verify_name_characters from lbrynet.core import log_support from lbrynet.core import utils -from lbrynet.core.LBRYMetadata import verify_name_characters from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, \ diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 2b1aa4b6d..9b448c40f 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -11,7 +11,7 @@ from twisted.internet.task import LoopingCall from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob -from lbrynet.core.LBRYMetadata import LBRYFeeValidator +from lbrynet.metadata.LBRYFee import LBRYFeeValidator from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py index 0af938954..10d6b48eb 100644 --- a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -6,7 +6,7 @@ import googlefinance from twisted.internet import defer, reactor from twisted.internet.task import LoopingCall -from lbrynet.core.LBRYMetadata import LBRYFeeValidator +from lbrynet.metadata.LBRYFee import LBRYFeeValidator log = logging.getLogger(__name__) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index de2dca070..9db95381a 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -9,7 +9,7 @@ from lbrynet.core.Error import InsufficientFundsError from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.core.PaymentRateManager import PaymentRateManager -from lbrynet.core.LBRYMetadata import Metadata +from lbrynet.metadata.LBRYMetadata import Metadata from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader from lbrynet.conf import LOG_FILE_NAME from twisted.internet import threads, defer diff --git a/lbrynet/metadata/LBRYFee.py b/lbrynet/metadata/LBRYFee.py new file mode 100644 index 000000000..7954135f7 --- /dev/null +++ b/lbrynet/metadata/LBRYFee.py @@ -0,0 +1,116 @@ +import logging + +from lbrynet.metadata.Validator import Validator, skip_validate +from lbrynet.conf import CURRENCIES + +log = logging.getLogger(__name__) + + +def verify_supported_currency(fee): + assert len(fee) == 1 + for c in fee: + assert c in CURRENCIES + return True + + +def verify_amount(x): + return isinstance(x, float) and x > 0 + + +class LBCFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount', verify_amount), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = [] + + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class BTCFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount',verify_amount), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = [] + + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class USDFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount',verify_amount), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = [] + + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class LBRYFeeValidator(Validator): + CV001 = "0.0.1" + CURRENT_CURRENCY_VERSION = CV001 + + CURRENCY_REVISIONS = {} + + CURRENCY_REVISIONS[CV001] = [ + (Validator.OPTIONAL, 'BTC', BTCFeeValidator.validate), + (Validator.OPTIONAL, 'USD', USDFeeValidator.validate), + (Validator.OPTIONAL, 'LBC', LBCFeeValidator.validate), + ] + + CURRENCY_MIGRATIONS = [] + + current_version = CURRENT_CURRENCY_VERSION + versions = CURRENCY_REVISIONS + migrations = CURRENCY_MIGRATIONS + + def __init__(self, fee_dict): + Validator.__init__(self, fee_dict) + self.currency_symbol = self.keys()[0] + self.amount = self._get_amount() + self.address = self[self.currency_symbol]['address'] + + def _get_amount(self): + amt = self[self.currency_symbol]['amount'] + if isinstance(amt, float): + return amt + else: + try: + return float(amt) + except TypeError: + log.error('Failed to convert %s to float', amt) + raise diff --git a/lbrynet/metadata/LBRYMetadata.py b/lbrynet/metadata/LBRYMetadata.py new file mode 100644 index 000000000..3bd477ee1 --- /dev/null +++ b/lbrynet/metadata/LBRYMetadata.py @@ -0,0 +1,86 @@ +import logging + +from lbrynet.metadata.Validator import Validator, skip_validate +from lbrynet.metadata.LBRYFee import LBRYFeeValidator, verify_supported_currency +from lbrynet.conf import SOURCE_TYPES + +log = logging.getLogger(__name__) +NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-' + + +def verify_name_characters(name): + for c in name: + assert c in NAME_ALLOWED_CHARSET, "Invalid character" + return True + + +def validate_sources(sources): + for source in sources: + assert source in SOURCE_TYPES, "Unknown source type: %s" % str(source) + return True + + +class Metadata(Validator): + MV001 = "0.0.1" + MV002 = "0.0.2" + MV003 = "0.0.3" + CURRENT_METADATA_VERSION = MV003 + + METADATA_REVISIONS = {} + + METADATA_REVISIONS[MV001] = [ + (Validator.REQUIRE, 'title', skip_validate), + (Validator.REQUIRE, 'description', skip_validate), + (Validator.REQUIRE, 'author', skip_validate), + (Validator.REQUIRE, 'language', skip_validate), + (Validator.REQUIRE, 'license', skip_validate), + (Validator.REQUIRE, 'content-type', skip_validate), + (Validator.REQUIRE, 'sources', validate_sources), + (Validator.OPTIONAL, 'thumbnail', skip_validate), + (Validator.OPTIONAL, 'preview', skip_validate), + (Validator.OPTIONAL, 'fee', verify_supported_currency), + (Validator.OPTIONAL, 'contact', skip_validate), + (Validator.OPTIONAL, 'pubkey', skip_validate), + ] + + METADATA_REVISIONS[MV002] = [ + (Validator.REQUIRE, 'nsfw', skip_validate), + (Validator.REQUIRE, 'ver', skip_validate), + (Validator.OPTIONAL, 'license_url', skip_validate), + ] + + METADATA_REVISIONS[MV003] = [ + (Validator.REQUIRE, 'content_type', skip_validate), + (Validator.SKIP, 'content-type'), + (Validator.OPTIONAL, 'sig', skip_validate), + (Validator.IF_KEY, 'sig', (Validator.REQUIRE, 'pubkey', skip_validate), Validator.DO_NOTHING), + (Validator.IF_KEY, 'pubkey', (Validator.REQUIRE, 'sig', skip_validate), Validator.DO_NOTHING), + ] + + MIGRATE_MV001_TO_MV002 = [ + (Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.LOAD, 'nsfw', False)), + (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.LOAD, 'ver', MV002)), + ] + + MIGRATE_MV002_TO_MV003 = [ + (Validator.IF_KEY, 'content-type', (Validator.UPDATE, 'content-type', 'content_type'), Validator.DO_NOTHING), + (Validator.IF_VAL, 'ver', MV002, (Validator.LOAD, 'ver', MV003), Validator.DO_NOTHING), + ] + + METADATA_MIGRATIONS = [ + MIGRATE_MV001_TO_MV002, + MIGRATE_MV002_TO_MV003, + ] + + current_version = CURRENT_METADATA_VERSION + versions = METADATA_REVISIONS + migrations = METADATA_MIGRATIONS + + def __init__(self, metadata, process_now=True): + Validator.__init__(self, metadata, process_now) + self.meta_version = self.get('ver', Metadata.MV001) + self._load_fee() + + def _load_fee(self): + if 'fee' in self: + self.update({'fee': LBRYFeeValidator(self['fee'])}) diff --git a/lbrynet/metadata/Validator.py b/lbrynet/metadata/Validator.py new file mode 100644 index 000000000..b08ed64e2 --- /dev/null +++ b/lbrynet/metadata/Validator.py @@ -0,0 +1,155 @@ +import json +import logging +from copy import deepcopy +from distutils.version import StrictVersion +from lbrynet.core.utils import version_is_greater_than + +log = logging.getLogger(__name__) + + +def skip_validate(value): + return True + + +def processor(cls): + for methodname in dir(cls): + method = getattr(cls, methodname) + if hasattr(method, 'cmd_name'): + cls.commands.update({method.cmd_name: methodname}) + return cls + + +def cmd(cmd_name): + def wrapper(func): + func.cmd_name = cmd_name + return func + return wrapper + + +@processor +class Validator(dict): + """ + Base class for validated dictionaries + """ + + # override these + current_version = None + versions = {} + migrations = [] + + # built in commands + DO_NOTHING = "do_nothing" + UPDATE = "update_key" + IF_KEY = "if_key" + REQUIRE = "require" + SKIP = "skip" + OPTIONAL = "optional" + LOAD = "load" + IF_VAL = "if_val" + + commands = {} + + @classmethod + def load_from_hex(cls, hex_val): + return cls(json.loads(hex_val.decode('hex'))) + + @classmethod + def validate(cls, value): + if cls(value): + return True + else: + return False + + def __init__(self, value, process_now=False): + dict.__init__(self) + self._skip = [] + value_to_load = deepcopy(value) + if process_now: + self.process(value_to_load) + self._verify_value(value_to_load) + self.version = self.get('ver', "0.0.1") + + def process(self, value): + if self.migrations is not None: + self._migrate_value(value) + + @cmd(DO_NOTHING) + def _do_nothing(self): + pass + + @cmd(SKIP) + def _add_to_skipped(self, rx_value, key): + if key not in self._skip: + self._skip.append(key) + + @cmd(UPDATE) + def _update(self, rx_value, old_key, new_key): + rx_value.update({new_key: rx_value.pop(old_key)}) + + @cmd(IF_KEY) + def _if_key(self, rx_value, key, if_true, if_else): + if key in rx_value: + return self._handle(if_true, rx_value) + return self._handle(if_else, rx_value) + + @cmd(IF_VAL) + def _if_val(self, rx_value, key, val, if_true, if_else): + if key in rx_value: + if rx_value[key] == val: + return self._handle(if_true, rx_value) + return self._handle(if_else, rx_value) + + @cmd(LOAD) + def _load(self, rx_value, key, value): + rx_value.update({key: value}) + + @cmd(REQUIRE) + def _require(self, rx_value, key, validator=None): + if key not in self._skip: + assert key in rx_value, "Key is missing: %s" % key + if isinstance(validator, type): + assert isinstance(rx_value[key], validator), "%s: %s isn't required %s" % (key, type(rx_value[key]), validator) + elif callable(validator): + assert validator(rx_value[key]), "Failed to validate %s" % key + self.update({key: rx_value.pop(key)}) + + @cmd(OPTIONAL) + def _optional(self, rx_value, key, validator=None): + if key in rx_value and key not in self._skip: + if isinstance(validator, type): + assert isinstance(rx_value[key], validator), "%s type %s isn't required %s" % (key, type(rx_value[key]), validator) + elif callable(validator): + assert validator(rx_value[key]), "Failed to validate %s" % key + self.update({key: rx_value.pop(key)}) + + def _handle(self, cmd_tpl, value): + if cmd_tpl == Validator.DO_NOTHING: + return + command = cmd_tpl[0] + f = getattr(self, self.commands[command]) + if len(cmd_tpl) > 1: + args = (value,) + cmd_tpl[1:] + f(*args) + else: + f() + + def _load_revision(self, version, value): + for k in self.versions[version]: + self._handle(k, value) + + def _verify_value(self, value): + val_ver = value.get('ver', "0.0.1") + # verify version requirements in reverse order starting from the version asserted in the value + versions = sorted([v for v in self.versions if not version_is_greater_than(v, val_ver)], key=StrictVersion, reverse=True) + for version in versions: + self._load_revision(version, value) + assert value == {} or value.keys() == self._skip, "Unknown keys: %s" % json.dumps(value) + + def _migrate_value(self, value): + for migration in self.migrations: + self._run_migration(migration, value) + + def _run_migration(self, commands, value): + for cmd in commands: + self._handle(cmd, value) + diff --git a/lbrynet/metadata/__init__.py b/lbrynet/metadata/__init__.py new file mode 100644 index 000000000..e69de29bb From c5706fa28ba4f2f67f920e09d05c85bcdb07e429 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 18 Aug 2016 00:38:38 -0400 Subject: [PATCH 9/9] fix imports in tests --- tests/lbrynet/core/test_LBRYExchangeRateManager.py | 2 +- tests/lbrynet/core/test_LBRYMetadata.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lbrynet/core/test_LBRYExchangeRateManager.py b/tests/lbrynet/core/test_LBRYExchangeRateManager.py index 2a6457536..3ff4c99d8 100644 --- a/tests/lbrynet/core/test_LBRYExchangeRateManager.py +++ b/tests/lbrynet/core/test_LBRYExchangeRateManager.py @@ -1,5 +1,5 @@ import mock -from lbrynet.core import LBRYMetadata +from lbrynet.metadata import LBRYMetadata from lbrynet.lbrynet_daemon import LBRYExchangeRateManager from twisted.trial import unittest diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index 953b8221f..f76a05f4e 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -1,4 +1,4 @@ -from lbrynet.core import LBRYMetadata +from lbrynet.metadata import LBRYMetadata from twisted.trial import unittest