change content-type to content_type, make changing metadata easier

lbrynet/core/LBRYMetadata.py

@@ -1,36 +1,13 @@
 import json
-
+import logging
 from copy import deepcopy
 from lbrynet.conf import CURRENCIES
-from lbrynet.core import utils
+from distutils.version import StrictVersion
-import logging
 
 log = logging.getLogger(__name__)
 
-BITTREX_FEE = 0.0025
-
-# Metadata version
 SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
 NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-'
-BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources']
-OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey']
-
-MV001 = "0.0.1"
-MV002 = "0.0.2"
-CURRENT_METADATA_VERSION = MV002
-
-METADATA_REVISIONS = {}
-METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}
-METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']}
-
-# Fee version
-BASE_FEE_FIELDS = ['amount', 'address']
-
-FV001 = "0.0.1"
-CURRENT_FEE_REVISION = FV001
-
-FEE_REVISIONS = {}
-FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []}
 
 
 def verify_name_characters(name):
@@ -39,18 +16,238 @@ def verify_name_characters(name):
     return True
 
 
-class LBRYFeeValidator(dict):
+def skip_validate(value):
-    def __init__(self, fee_dict):
+    pass
+
+
+def verify_supported_currency(fee):
+    assert len(fee) == 1
+    for c in fee:
+        assert c in CURRENCIES
+
+
+def validate_sources(sources):
+    for source in sources:
+        assert source in SOURCE_TYPES, "Unknown source type"
+
+
+class Validator(dict):
+    """
+    Base class for validated dictionaries
+    """
+
+    DO_NOTHING = "pass"
+    UPDATE = "update_key"
+    IF_KEY = "if_key_exists"
+    REQUIRE = "require"
+    SKIP = "skip"
+    OPTIONAL = "add_optional"
+    ADD = "add"
+    IF_VAL = "if_val"
+    SUPERSEDE = "supersede"
+
+    # override these
+    current_version = None
+    versions = None
+    migrations = None
+    supersessions = None
+
+    @classmethod
+    def load_from_hex(cls, hex_val):
+        return cls(json.loads(hex_val.decode('hex')))
+
+    def process(self):
+        unprocessed = deepcopy(self)
+        if self.migrations is not None:
+            self._migrate_value(unprocessed)
+
+    def serialize(self):
+        return json.dumps(self).encode("hex")
+
+    def as_json(self):
+        return json.dumps(self)
+
+    def __init__(self, value, process_now=True):
         dict.__init__(self)
-        assert len(fee_dict) == 1
+        self._skip = []
-        self.fee_version = None
+        value_to_load = deepcopy(value)
-        self.currency_symbol = None
+        if self.supersessions is not None:
+            self._run_supersessions(value_to_load)
+        self._verify_value(value_to_load)
+        self._raw = deepcopy(self)
+        self.version = self.get('ver', "0.0.1")
+        if process_now:
+            self.process()
 
-        fee_to_load = deepcopy(fee_dict)
+    def _handle(self, cmd_tpl, value):
+        if cmd_tpl == self.DO_NOTHING:
+            return
 
-        for currency in fee_dict:
+        cmd = cmd_tpl[0]
-            self._verify_fee(currency, fee_to_load)
 
+        if cmd == self.IF_KEY:
+            key, on_key, on_else = cmd_tpl[1:]
+            if key in value:
+                return self._handle(on_key, value)
+            elif on_else:
+                return self._handle(on_else, value)
+            return
+
+        elif cmd == self.IF_VAL:
+            key, v, on_val, on_else = cmd_tpl[1:]
+            if key not in value:
+                return self._handle(on_else, value)
+            if value[key] == v:
+                return self._handle(on_val, value)
+            elif on_else:
+                return self._handle(on_else, value)
+            return
+
+        elif cmd == self.UPDATE:
+            old_key, new_key = cmd_tpl[1:]
+            value.update({new_key: value.pop(old_key)})
+
+        elif cmd == self.REQUIRE:
+            required, validator = cmd_tpl[1:]
+            if required not in self._skip:
+                assert required in value if required not in self else True, "Missing required field: %s, %s" % (required, self.as_json())
+                if required not in self._skip and required in value:
+                    self.update({required: value.pop(required)})
+                    validator(self[required])
+            else:
+                pass
+
+        elif cmd == self.OPTIONAL:
+            optional, validator = cmd_tpl[1:]
+            if optional in value and optional not in self._skip:
+                self.update({optional: value.pop(optional)})
+                validator(self[optional])
+            else:
+                pass
+
+        elif cmd == self.SKIP:
+            to_skip = cmd_tpl[1]
+            self._skip.append(to_skip)
+
+        elif cmd == self.ADD:
+            key, pushed_val = cmd_tpl[1:]
+            self.update({key: pushed_val})
+
+        elif cmd == self.SUPERSEDE:
+            ver = cmd_tpl[1]
+            self.update({'ver': ver})
+            self.version = ver
+
+    def _load_revision(self, version, value):
+        for k in self.versions[version]:
+            self._handle(k, value)
+
+    def _verify_value(self, value):
+        for version in sorted(self.versions, key=StrictVersion):
+            self._load_revision(version, value)
+            if not value:
+                self['ver'] = version
+                break
+        for skip in self._skip:
+            if skip in value:
+                value.pop(skip)
+        assert value == {}, "Unknown keys: %s, %s" % (json.dumps(value.keys()), self.as_json())
+
+    def _migrate_value(self, value):
+        for migration in self.migrations:
+            self._run_migration(migration, value)
+
+    def _run_migration(self, commands, value):
+        for cmd in commands:
+            self._handle(cmd, value)
+
+    def _run_supersessions(self, value):
+        for cmd in self.supersessions:
+            self._handle(cmd, value)
+
+
+class LBCFeeValidator(Validator):
+    FV001 = "0.0.1"
+    CURRENT_FEE_VERSION = FV001
+
+    FEE_REVISIONS = {}
+
+    FEE_REVISIONS[FV001] = [
+        (Validator.REQUIRE, 'amount', skip_validate),
+        (Validator.REQUIRE, 'address', skip_validate),
+    ]
+
+    FEE_MIGRATIONS = None
+
+    current_version = CURRENT_FEE_VERSION
+    versions = FEE_REVISIONS
+    migrations = FEE_MIGRATIONS
+
+    def __init__(self, fee):
+        Validator.__init__(self, fee)
+
+
+class BTCFeeValidator(Validator):
+    FV001 = "0.0.1"
+    CURRENT_FEE_VERSION = FV001
+
+    FEE_REVISIONS = {}
+
+    FEE_REVISIONS[FV001] = [
+        (Validator.REQUIRE, 'amount', skip_validate),
+        (Validator.REQUIRE, 'address', skip_validate),
+    ]
+
+    FEE_MIGRATIONS = None
+    current_version = CURRENT_FEE_VERSION
+    versions = FEE_REVISIONS
+    migrations = FEE_MIGRATIONS
+
+    def __init__(self, fee):
+        Validator.__init__(self, fee)
+
+
+class USDFeeValidator(Validator):
+    FV001 = "0.0.1"
+    CURRENT_FEE_VERSION = FV001
+
+    FEE_REVISIONS = {}
+
+    FEE_REVISIONS[FV001] = [
+        (Validator.REQUIRE, 'amount', skip_validate),
+        (Validator.REQUIRE, 'address', skip_validate),
+    ]
+
+    FEE_MIGRATIONS = None
+    current_version = CURRENT_FEE_VERSION
+    versions = FEE_REVISIONS
+    migrations = FEE_MIGRATIONS
+
+    def __init__(self, fee):
+        Validator.__init__(self, fee)
+
+
+class LBRYFeeValidator(Validator):
+    CV001 = "0.0.1"
+    CURRENT_CURRENCY_VERSION = CV001
+
+    CURRENCY_REVISIONS = {}
+
+    CURRENCY_REVISIONS[CV001] = [
+        (Validator.OPTIONAL, 'BTC', BTCFeeValidator),
+        (Validator.OPTIONAL, 'USD', USDFeeValidator),
+        (Validator.OPTIONAL, 'LBC', LBCFeeValidator),
+    ]
+
+    CURRENCY_MIGRATIONS = None
+
+    current_version = CURRENT_CURRENCY_VERSION
+    versions = CURRENCY_REVISIONS
+    migrations = CURRENCY_MIGRATIONS
+
+    def __init__(self, fee_dict):
+        Validator.__init__(self, fee_dict)
+        self.currency_symbol = self.keys()[0]
         self.amount = self._get_amount()
         self.address = self[self.currency_symbol]['address']
 
@@ -65,71 +262,73 @@ class LBRYFeeValidator(dict):
                 log.error('Failed to convert %s to float', amt)
                 raise
 
-    def _verify_fee(self, currency, f):
-        # str in case someone made a claim with a wierd fee
-        assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency)
-        self.currency_symbol = currency
-        self.update({currency: {}})
-        for version in FEE_REVISIONS:
-            self._load_revision(version, f)
-            if not f:
-                self.fee_version = version
-                break
-        assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys())
 
-    def _load_revision(self, version, f):
+class Metadata(Validator):
-        for k in FEE_REVISIONS[version]['required']:
+    MV001 = "0.0.1"
-            assert k in f[self.currency_symbol], "Missing required fee field: %s" % k
+    MV002 = "0.0.2"
-            self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
+    MV003 = "0.0.3"
-        for k in FEE_REVISIONS[version]['optional']:
+    CURRENT_METADATA_VERSION = MV003
-            if k in f[self.currency_symbol]:
-                self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
 
+    METADATA_REVISIONS = {}
 
-class Metadata(dict):
+    METADATA_REVISIONS[MV001] = [
-    @classmethod
+        (Validator.REQUIRE, 'title', skip_validate),
-    def load_from_hex(cls, metadata):
+        (Validator.REQUIRE, 'description', skip_validate),
-        return cls(json.loads(metadata.decode('hex')))
+        (Validator.REQUIRE, 'author', skip_validate),
+        (Validator.REQUIRE, 'language', skip_validate),
+        (Validator.REQUIRE, 'license', skip_validate),
+        (Validator.REQUIRE, 'content-type', skip_validate),
+        (Validator.REQUIRE, 'sources', validate_sources),
+        (Validator.OPTIONAL, 'thumbnail', skip_validate),
+        (Validator.OPTIONAL, 'preview', skip_validate),
+        (Validator.OPTIONAL, 'fee', verify_supported_currency),
+        (Validator.OPTIONAL, 'contact', skip_validate),
+        (Validator.OPTIONAL, 'pubkey', skip_validate),
+    ]
+
+    METADATA_REVISIONS[MV002] = [
+        (Validator.REQUIRE, 'nsfw', skip_validate),
+        (Validator.REQUIRE, 'ver', skip_validate),
+        (Validator.OPTIONAL, 'license_url', skip_validate),
+    ]
+
+    METADATA_REVISIONS[MV003] = [
+        (Validator.REQUIRE, 'content_type', skip_validate),
+        (Validator.SKIP, 'content-type'),
+        (Validator.OPTIONAL, 'sig', skip_validate),
+        (Validator.IF_KEY, 'sig', (Validator.REQUIRE, 'pubkey', skip_validate), Validator.DO_NOTHING),
+        (Validator.IF_KEY, 'pubkey', (Validator.REQUIRE, 'sig', skip_validate), Validator.DO_NOTHING),
+    ]
+
+    MIGRATE_MV001_TO_MV002 = [
+        (Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.ADD, 'nsfw', False)),
+        (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV002)),
+    ]
+    MIGRATE_MV002_TO_MV003 = [
+        (Validator.IF_KEY, 'content_type', Validator.DO_NOTHING, (Validator.UPDATE, 'content-type', 'content_type')),
+        (Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV003)),
+        (Validator.IF_VAL, 'ver', MV002, (Validator.SUPERSEDE, MV003), Validator.DO_NOTHING),
+    ]
+
+    METADATA_MIGRATIONS = [
+        MIGRATE_MV001_TO_MV002,
+        MIGRATE_MV002_TO_MV003,
+    ]
+
+    SUPERSESSIONS = [
+        (Validator.SKIP, 'content-type'),
+    ]
+
+    current_version = CURRENT_METADATA_VERSION
+    versions = METADATA_REVISIONS
+    migrations = METADATA_MIGRATIONS
+    supersessions = SUPERSESSIONS
 
     def __init__(self, metadata):
-        dict.__init__(self)
+        Validator.__init__(self, metadata)
-        self.meta_version = None
+        self.meta_version = self.get('ver', Metadata.MV001)
-        metadata_to_load = deepcopy(metadata)
+        self._load_fee()
 
-        self._verify_sources(metadata_to_load)
+    def _load_fee(self):
-        self._verify_metadata(metadata_to_load)
+        if 'fee' in self:
-
+            self.update({'fee': LBRYFeeValidator(self['fee'])})
-    def _load_revision(self, version, metadata):
-        for k in METADATA_REVISIONS[version]['required']:
-            assert k in metadata, "Missing required metadata field: %s" % k
-            self.update({k: metadata.pop(k)})
-        for k in METADATA_REVISIONS[version]['optional']:
-            if k == 'fee':
-                self._load_fee(metadata)
-            elif k in metadata:
-                self.update({k: metadata.pop(k)})
-
-    def _load_fee(self, metadata):
-        if 'fee' in metadata:
-            self['fee'] = LBRYFeeValidator(metadata.pop('fee'))
-
-    def _verify_sources(self, metadata):
-        assert "sources" in metadata, "No sources given"
-        for source in metadata['sources']:
-            assert source in SOURCE_TYPES, "Unknown source type"
-
-    def _verify_metadata(self, metadata):
-        for version in METADATA_REVISIONS:
-            self._load_revision(version, metadata)
-            if not metadata:
-                self.meta_version = version
-                if utils.version_is_greater_than(self.meta_version, "0.0.1"):
-                    assert self.meta_version == self['ver'], "version mismatch"
-                break
-        assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys())
-
-    def serialize(self):
-        return json.dumps(self).encode("hex")
-
-    def as_json(self):
-        return json.dumps(self)

lbrynet/lbrynet_daemon/LBRYPublisher.py

@@ -9,7 +9,7 @@ from lbrynet.core.Error import InsufficientFundsError
 from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
 from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
 from lbrynet.core.PaymentRateManager import PaymentRateManager
-from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION
+from lbrynet.core.LBRYMetadata import Metadata
 from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
 from lbrynet.conf import LOG_FILE_NAME
 from twisted.internet import threads, defer
@@ -101,9 +101,9 @@ class Publisher(object):
         return d
 
     def _claim_name(self):
-        self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory,
+        self.metadata['content_type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory,
                                                                           self.lbry_file.file_name))[0]
-        self.metadata['ver'] = CURRENT_METADATA_VERSION
+        self.metadata['ver'] = Metadata.current_version
         m = Metadata(self.metadata)
 
         def set_tx_hash(txid):

tests/lbrynet/core/test_LBRYMetadata.py

@@ -21,7 +21,7 @@ class MetadataTest(unittest.TestCase):
             'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg'
         }
         m = LBRYMetadata.Metadata(metadata)
-        self.assertFalse('key' in m)
+        self.assertFalse('fee' in m)
 
     def test_assertion_if_invalid_source(self):
         metadata = {