diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py index 5c101078b..0f6daa537 100644 --- a/lbrynet/core/LBRYMetadata.py +++ b/lbrynet/core/LBRYMetadata.py @@ -1,36 +1,13 @@ import json - +import logging from copy import deepcopy from lbrynet.conf import CURRENCIES -from lbrynet.core import utils -import logging +from distutils.version import StrictVersion log = logging.getLogger(__name__) -BITTREX_FEE = 0.0025 - -# Metadata version SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-' -BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] -OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] - -MV001 = "0.0.1" -MV002 = "0.0.2" -CURRENT_METADATA_VERSION = MV002 - -METADATA_REVISIONS = {} -METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS} -METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']} - -# Fee version -BASE_FEE_FIELDS = ['amount', 'address'] - -FV001 = "0.0.1" -CURRENT_FEE_REVISION = FV001 - -FEE_REVISIONS = {} -FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []} def verify_name_characters(name): @@ -39,18 +16,238 @@ def verify_name_characters(name): return True -class LBRYFeeValidator(dict): - def __init__(self, fee_dict): +def skip_validate(value): + pass + + +def verify_supported_currency(fee): + assert len(fee) == 1 + for c in fee: + assert c in CURRENCIES + + +def validate_sources(sources): + for source in sources: + assert source in SOURCE_TYPES, "Unknown source type" + + +class Validator(dict): + """ + Base class for validated dictionaries + """ + + DO_NOTHING = "pass" + UPDATE = "update_key" + IF_KEY = "if_key_exists" + REQUIRE = "require" + SKIP = "skip" + OPTIONAL = "add_optional" + ADD = "add" + IF_VAL = "if_val" + SUPERSEDE = "supersede" + + # override these + current_version = None + versions = None + migrations = None + supersessions = None + + @classmethod + def load_from_hex(cls, hex_val): + return cls(json.loads(hex_val.decode('hex'))) + + def process(self): + unprocessed = deepcopy(self) + if self.migrations is not None: + self._migrate_value(unprocessed) + + def serialize(self): + return json.dumps(self).encode("hex") + + def as_json(self): + return json.dumps(self) + + def __init__(self, value, process_now=True): dict.__init__(self) - assert len(fee_dict) == 1 - self.fee_version = None - self.currency_symbol = None + self._skip = [] + value_to_load = deepcopy(value) + if self.supersessions is not None: + self._run_supersessions(value_to_load) + self._verify_value(value_to_load) + self._raw = deepcopy(self) + self.version = self.get('ver', "0.0.1") + if process_now: + self.process() - fee_to_load = deepcopy(fee_dict) + def _handle(self, cmd_tpl, value): + if cmd_tpl == self.DO_NOTHING: + return - for currency in fee_dict: - self._verify_fee(currency, fee_to_load) + cmd = cmd_tpl[0] + if cmd == self.IF_KEY: + key, on_key, on_else = cmd_tpl[1:] + if key in value: + return self._handle(on_key, value) + elif on_else: + return self._handle(on_else, value) + return + + elif cmd == self.IF_VAL: + key, v, on_val, on_else = cmd_tpl[1:] + if key not in value: + return self._handle(on_else, value) + if value[key] == v: + return self._handle(on_val, value) + elif on_else: + return self._handle(on_else, value) + return + + elif cmd == self.UPDATE: + old_key, new_key = cmd_tpl[1:] + value.update({new_key: value.pop(old_key)}) + + elif cmd == self.REQUIRE: + required, validator = cmd_tpl[1:] + if required not in self._skip: + assert required in value if required not in self else True, "Missing required field: %s, %s" % (required, self.as_json()) + if required not in self._skip and required in value: + self.update({required: value.pop(required)}) + validator(self[required]) + else: + pass + + elif cmd == self.OPTIONAL: + optional, validator = cmd_tpl[1:] + if optional in value and optional not in self._skip: + self.update({optional: value.pop(optional)}) + validator(self[optional]) + else: + pass + + elif cmd == self.SKIP: + to_skip = cmd_tpl[1] + self._skip.append(to_skip) + + elif cmd == self.ADD: + key, pushed_val = cmd_tpl[1:] + self.update({key: pushed_val}) + + elif cmd == self.SUPERSEDE: + ver = cmd_tpl[1] + self.update({'ver': ver}) + self.version = ver + + def _load_revision(self, version, value): + for k in self.versions[version]: + self._handle(k, value) + + def _verify_value(self, value): + for version in sorted(self.versions, key=StrictVersion): + self._load_revision(version, value) + if not value: + self['ver'] = version + break + for skip in self._skip: + if skip in value: + value.pop(skip) + assert value == {}, "Unknown keys: %s, %s" % (json.dumps(value.keys()), self.as_json()) + + def _migrate_value(self, value): + for migration in self.migrations: + self._run_migration(migration, value) + + def _run_migration(self, commands, value): + for cmd in commands: + self._handle(cmd, value) + + def _run_supersessions(self, value): + for cmd in self.supersessions: + self._handle(cmd, value) + + +class LBCFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = None + + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class BTCFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = None + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class USDFeeValidator(Validator): + FV001 = "0.0.1" + CURRENT_FEE_VERSION = FV001 + + FEE_REVISIONS = {} + + FEE_REVISIONS[FV001] = [ + (Validator.REQUIRE, 'amount', skip_validate), + (Validator.REQUIRE, 'address', skip_validate), + ] + + FEE_MIGRATIONS = None + current_version = CURRENT_FEE_VERSION + versions = FEE_REVISIONS + migrations = FEE_MIGRATIONS + + def __init__(self, fee): + Validator.__init__(self, fee) + + +class LBRYFeeValidator(Validator): + CV001 = "0.0.1" + CURRENT_CURRENCY_VERSION = CV001 + + CURRENCY_REVISIONS = {} + + CURRENCY_REVISIONS[CV001] = [ + (Validator.OPTIONAL, 'BTC', BTCFeeValidator), + (Validator.OPTIONAL, 'USD', USDFeeValidator), + (Validator.OPTIONAL, 'LBC', LBCFeeValidator), + ] + + CURRENCY_MIGRATIONS = None + + current_version = CURRENT_CURRENCY_VERSION + versions = CURRENCY_REVISIONS + migrations = CURRENCY_MIGRATIONS + + def __init__(self, fee_dict): + Validator.__init__(self, fee_dict) + self.currency_symbol = self.keys()[0] self.amount = self._get_amount() self.address = self[self.currency_symbol]['address'] @@ -65,71 +262,73 @@ class LBRYFeeValidator(dict): log.error('Failed to convert %s to float', amt) raise - def _verify_fee(self, currency, f): - # str in case someone made a claim with a wierd fee - assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) - self.currency_symbol = currency - self.update({currency: {}}) - for version in FEE_REVISIONS: - self._load_revision(version, f) - if not f: - self.fee_version = version - break - assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) - def _load_revision(self, version, f): - for k in FEE_REVISIONS[version]['required']: - assert k in f[self.currency_symbol], "Missing required fee field: %s" % k - self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) - for k in FEE_REVISIONS[version]['optional']: - if k in f[self.currency_symbol]: - self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) +class Metadata(Validator): + MV001 = "0.0.1" + MV002 = "0.0.2" + MV003 = "0.0.3" + CURRENT_METADATA_VERSION = MV003 + METADATA_REVISIONS = {} -class Metadata(dict): - @classmethod - def load_from_hex(cls, metadata): - return cls(json.loads(metadata.decode('hex'))) + METADATA_REVISIONS[MV001] = [ + (Validator.REQUIRE, 'title', skip_validate), + (Validator.REQUIRE, 'description', skip_validate), + (Validator.REQUIRE, 'author', skip_validate), + (Validator.REQUIRE, 'language', skip_validate), + (Validator.REQUIRE, 'license', skip_validate), + (Validator.REQUIRE, 'content-type', skip_validate), + (Validator.REQUIRE, 'sources', validate_sources), + (Validator.OPTIONAL, 'thumbnail', skip_validate), + (Validator.OPTIONAL, 'preview', skip_validate), + (Validator.OPTIONAL, 'fee', verify_supported_currency), + (Validator.OPTIONAL, 'contact', skip_validate), + (Validator.OPTIONAL, 'pubkey', skip_validate), + ] + + METADATA_REVISIONS[MV002] = [ + (Validator.REQUIRE, 'nsfw', skip_validate), + (Validator.REQUIRE, 'ver', skip_validate), + (Validator.OPTIONAL, 'license_url', skip_validate), + ] + + METADATA_REVISIONS[MV003] = [ + (Validator.REQUIRE, 'content_type', skip_validate), + (Validator.SKIP, 'content-type'), + (Validator.OPTIONAL, 'sig', skip_validate), + (Validator.IF_KEY, 'sig', (Validator.REQUIRE, 'pubkey', skip_validate), Validator.DO_NOTHING), + (Validator.IF_KEY, 'pubkey', (Validator.REQUIRE, 'sig', skip_validate), Validator.DO_NOTHING), + ] + + MIGRATE_MV001_TO_MV002 = [ + (Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.ADD, 'nsfw', False)), + (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV002)), + ] + MIGRATE_MV002_TO_MV003 = [ + (Validator.IF_KEY, 'content_type', Validator.DO_NOTHING, (Validator.UPDATE, 'content-type', 'content_type')), + (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV003)), + (Validator.IF_VAL, 'ver', MV002, (Validator.SUPERSEDE, MV003), Validator.DO_NOTHING), + ] + + METADATA_MIGRATIONS = [ + MIGRATE_MV001_TO_MV002, + MIGRATE_MV002_TO_MV003, + ] + + SUPERSESSIONS = [ + (Validator.SKIP, 'content-type'), + ] + + current_version = CURRENT_METADATA_VERSION + versions = METADATA_REVISIONS + migrations = METADATA_MIGRATIONS + supersessions = SUPERSESSIONS def __init__(self, metadata): - dict.__init__(self) - self.meta_version = None - metadata_to_load = deepcopy(metadata) + Validator.__init__(self, metadata) + self.meta_version = self.get('ver', Metadata.MV001) + self._load_fee() - self._verify_sources(metadata_to_load) - self._verify_metadata(metadata_to_load) - - def _load_revision(self, version, metadata): - for k in METADATA_REVISIONS[version]['required']: - assert k in metadata, "Missing required metadata field: %s" % k - self.update({k: metadata.pop(k)}) - for k in METADATA_REVISIONS[version]['optional']: - if k == 'fee': - self._load_fee(metadata) - elif k in metadata: - self.update({k: metadata.pop(k)}) - - def _load_fee(self, metadata): - if 'fee' in metadata: - self['fee'] = LBRYFeeValidator(metadata.pop('fee')) - - def _verify_sources(self, metadata): - assert "sources" in metadata, "No sources given" - for source in metadata['sources']: - assert source in SOURCE_TYPES, "Unknown source type" - - def _verify_metadata(self, metadata): - for version in METADATA_REVISIONS: - self._load_revision(version, metadata) - if not metadata: - self.meta_version = version - if utils.version_is_greater_than(self.meta_version, "0.0.1"): - assert self.meta_version == self['ver'], "version mismatch" - break - assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys()) - - def serialize(self): - return json.dumps(self).encode("hex") - - def as_json(self): - return json.dumps(self) + def _load_fee(self): + if 'fee' in self: + self.update({'fee': LBRYFeeValidator(self['fee'])}) \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 75ceb6093..de2dca070 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -9,7 +9,7 @@ from lbrynet.core.Error import InsufficientFundsError from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.core.PaymentRateManager import PaymentRateManager -from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION +from lbrynet.core.LBRYMetadata import Metadata from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader from lbrynet.conf import LOG_FILE_NAME from twisted.internet import threads, defer @@ -101,9 +101,9 @@ class Publisher(object): return d def _claim_name(self): - self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, + self.metadata['content_type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name))[0] - self.metadata['ver'] = CURRENT_METADATA_VERSION + self.metadata['ver'] = Metadata.current_version m = Metadata(self.metadata) def set_tx_hash(txid): diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py index e5c3255b3..5a35c6507 100644 --- a/tests/lbrynet/core/test_LBRYMetadata.py +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -21,7 +21,7 @@ class MetadataTest(unittest.TestCase): 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' } m = LBRYMetadata.Metadata(metadata) - self.assertFalse('key' in m) + self.assertFalse('fee' in m) def test_assertion_if_invalid_source(self): metadata = {