Refactor Validator to new StructuredDict class that uses JSON Schema

- Uses JSON schema for all validation (so far no custom code needed)
 - Can migrate up and down with any versioning scheme
 - Does migrations with regular dictionary operations instead of a DSL

lbrynet/metadata/StructuredDict.py

@@ -0,0 +1,57 @@
+import jsonschema
+import logging
+
+from jsonschema import ValidationError
+
+log = logging.getLogger(__name__)
+
+
+class StructuredDict(dict):
+    """
+    A dictionary that enforces a structure specified by a schema, and supports
+    migration between different versions of the schema.
+    """
+
+    # To be specified in sub-classes, an array in the format
+    # [(version, schema, migration), ...]
+    _versions = []
+
+    # Used internally to allow schema lookups by version number
+    _schemas = {}
+
+    version = None
+
+    def __init__(self, value, starting_version, migrate=True, target_version=None):
+        dict.__init__(self, value)
+
+        self.version = starting_version
+        self._schemas = dict([(version, schema) for (version, schema, _) in self._versions])
+
+        self.validate(starting_version)
+
+        if migrate:
+            self.migrate(target_version)
+
+    def validate(self, version):
+        jsonschema.validate(self, self._schemas[version])
+
+    def migrate(self, target_version=None):
+        if target_version:
+            assert self._versions.index(target_version) > self.versions.index(self.version), "Current version is above target version"
+
+        above_starting_version = False
+        for version, schema, migration in self._versions:
+            if not above_starting_version:
+                if version == self.version:
+                    above_starting_version = True
+                continue
+
+            migration()
+            try:
+                self.validate(version)
+            except ValidationError as e:
+                raise ValidationError, "Could not migrate to version %s due to validation error: %s" % (version, e.message)
+
+            self.version = version
+            if target_version and version == target_version:
+                break

lbrynet/metadata/Validator.py

@@ -1,155 +0,0 @@
-import json
-import logging
-from copy import deepcopy
-from distutils.version import StrictVersion
-from lbrynet.core.utils import version_is_greater_than
-
-log = logging.getLogger(__name__)
-
-
-def skip_validate(value):
-    return True
-
-
-def processor(cls):
-    for methodname in dir(cls):
-        method = getattr(cls, methodname)
-        if hasattr(method, 'cmd_name'):
-            cls.commands.update({method.cmd_name: methodname})
-    return cls
-
-
-def cmd(cmd_name):
-    def wrapper(func):
-        func.cmd_name = cmd_name
-        return func
-    return wrapper
-
-
-@processor
-class Validator(dict):
-    """
-    Base class for validated dictionaries
-    """
-
-    # override these
-    current_version = None
-    versions = {}
-    migrations = []
-
-    # built in commands
-    DO_NOTHING = "do_nothing"
-    UPDATE = "update_key"
-    IF_KEY = "if_key"
-    REQUIRE = "require"
-    SKIP = "skip"
-    OPTIONAL = "optional"
-    LOAD = "load"
-    IF_VAL = "if_val"
-
-    commands = {}
-
-    @classmethod
-    def load_from_hex(cls, hex_val):
-        return cls(json.loads(hex_val.decode('hex')))
-
-    @classmethod
-    def validate(cls, value):
-        if cls(value):
-            return True
-        else:
-            return False
-
-    def __init__(self, value, process_now=False):
-        dict.__init__(self)
-        self._skip = []
-        value_to_load = deepcopy(value)
-        if process_now:
-            self.process(value_to_load)
-        self._verify_value(value_to_load)
-        self.version = self.get('ver', "0.0.1")
-
-    def process(self, value):
-        if self.migrations is not None:
-            self._migrate_value(value)
-
-    @cmd(DO_NOTHING)
-    def _do_nothing(self):
-        pass
-
-    @cmd(SKIP)
-    def _add_to_skipped(self, rx_value, key):
-        if key not in self._skip:
-            self._skip.append(key)
-
-    @cmd(UPDATE)
-    def _update(self, rx_value, old_key, new_key):
-        rx_value.update({new_key: rx_value.pop(old_key)})
-
-    @cmd(IF_KEY)
-    def _if_key(self, rx_value, key, if_true, if_else):
-        if key in rx_value:
-            return self._handle(if_true, rx_value)
-        return self._handle(if_else, rx_value)
-
-    @cmd(IF_VAL)
-    def _if_val(self, rx_value, key, val, if_true, if_else):
-        if key in rx_value:
-            if rx_value[key] == val:
-                return self._handle(if_true, rx_value)
-        return self._handle(if_else, rx_value)
-
-    @cmd(LOAD)
-    def _load(self, rx_value, key, value):
-        rx_value.update({key: value})
-
-    @cmd(REQUIRE)
-    def _require(self, rx_value, key, validator=None):
-        if key not in self._skip:
-            assert key in rx_value, "Key is missing: %s" % key
-            if isinstance(validator, type):
-                assert isinstance(rx_value[key], validator), "%s: %s isn't required %s" % (key, type(rx_value[key]), validator)
-            elif callable(validator):
-                assert validator(rx_value[key]), "Failed to validate %s" % key
-            self.update({key: rx_value.pop(key)})
-
-    @cmd(OPTIONAL)
-    def _optional(self, rx_value, key, validator=None):
-        if key in rx_value and key not in self._skip:
-            if isinstance(validator, type):
-                assert isinstance(rx_value[key], validator), "%s type %s isn't required %s" % (key, type(rx_value[key]), validator)
-            elif callable(validator):
-                assert validator(rx_value[key]), "Failed to validate %s" % key
-            self.update({key: rx_value.pop(key)})
-
-    def _handle(self, cmd_tpl, value):
-        if cmd_tpl == Validator.DO_NOTHING:
-            return
-        command = cmd_tpl[0]
-        f = getattr(self, self.commands[command])
-        if len(cmd_tpl) > 1:
-            args = (value,) + cmd_tpl[1:]
-            f(*args)
-        else:
-            f()
-
-    def _load_revision(self, version, value):
-        for k in self.versions[version]:
-            self._handle(k, value)
-
-    def _verify_value(self, value):
-        val_ver = value.get('ver', "0.0.1")
-        # verify version requirements in reverse order starting from the version asserted in the value
-        versions = sorted([v for v in self.versions if not version_is_greater_than(v, val_ver)], key=StrictVersion, reverse=True)
-        for version in versions:
-            self._load_revision(version, value)
-        assert value == {} or value.keys() == self._skip, "Unknown keys: %s" % json.dumps(value)
-
-    def _migrate_value(self, value):
-        for migration in self.migrations:
-            self._run_migration(migration, value)
-
-    def _run_migration(self, commands, value):
-        for cmd in commands:
-            self._handle(cmd, value)
-

setup.py

@@ -40,6 +40,7 @@ requires = [
     'lbryum',
     'jsonrpc',
     'simplejson',
+    'jsonschema',
     'appdirs',
     'six==1.9.0',
     'base58',