forked from LBRYCommunity/lbry-sdk
change content-type to content_type, make changing metadata easier
This commit is contained in:
Jack
parent
ff31ddd2d5
commit
272e074a15
3 changed files with 298 additions and 99 deletions
|
|
@ -1,36 +1,13 @@
|
|||
import json
|
||||
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from lbrynet.conf import CURRENCIES
|
||||
from lbrynet.core import utils
|
||||
import logging
|
||||
from distutils.version import StrictVersion
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
BITTREX_FEE = 0.0025
|
||||
|
||||
# Metadata version
|
||||
SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
|
||||
NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-'
|
||||
BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources']
|
||||
OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey']
|
||||
|
||||
MV001 = "0.0.1"
|
||||
MV002 = "0.0.2"
|
||||
CURRENT_METADATA_VERSION = MV002
|
||||
|
||||
METADATA_REVISIONS = {}
|
||||
METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}
|
||||
METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']}
|
||||
|
||||
# Fee version
|
||||
BASE_FEE_FIELDS = ['amount', 'address']
|
||||
|
||||
FV001 = "0.0.1"
|
||||
CURRENT_FEE_REVISION = FV001
|
||||
|
||||
FEE_REVISIONS = {}
|
||||
FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []}
|
||||
|
||||
|
||||
def verify_name_characters(name):
|
||||
|
|
@ -39,18 +16,238 @@ def verify_name_characters(name):
|
|||
return True
|
||||
|
||||
|
||||
class LBRYFeeValidator(dict):
|
||||
def __init__(self, fee_dict):
|
||||
def skip_validate(value):
|
||||
pass
|
||||
|
||||
|
||||
def verify_supported_currency(fee):
|
||||
assert len(fee) == 1
|
||||
for c in fee:
|
||||
assert c in CURRENCIES
|
||||
|
||||
|
||||
def validate_sources(sources):
|
||||
for source in sources:
|
||||
assert source in SOURCE_TYPES, "Unknown source type"
|
||||
|
||||
|
||||
class Validator(dict):
|
||||
"""
|
||||
Base class for validated dictionaries
|
||||
"""
|
||||
|
||||
DO_NOTHING = "pass"
|
||||
UPDATE = "update_key"
|
||||
IF_KEY = "if_key_exists"
|
||||
REQUIRE = "require"
|
||||
SKIP = "skip"
|
||||
OPTIONAL = "add_optional"
|
||||
ADD = "add"
|
||||
IF_VAL = "if_val"
|
||||
SUPERSEDE = "supersede"
|
||||
|
||||
# override these
|
||||
current_version = None
|
||||
versions = None
|
||||
migrations = None
|
||||
supersessions = None
|
||||
|
||||
@classmethod
|
||||
def load_from_hex(cls, hex_val):
|
||||
return cls(json.loads(hex_val.decode('hex')))
|
||||
|
||||
def process(self):
|
||||
unprocessed = deepcopy(self)
|
||||
if self.migrations is not None:
|
||||
self._migrate_value(unprocessed)
|
||||
|
||||
def serialize(self):
|
||||
return json.dumps(self).encode("hex")
|
||||
|
||||
def as_json(self):
|
||||
return json.dumps(self)
|
||||
|
||||
def __init__(self, value, process_now=True):
|
||||
dict.__init__(self)
|
||||
assert len(fee_dict) == 1
|
||||
self.fee_version = None
|
||||
self.currency_symbol = None
|
||||
self._skip = []
|
||||
value_to_load = deepcopy(value)
|
||||
if self.supersessions is not None:
|
||||
self._run_supersessions(value_to_load)
|
||||
self._verify_value(value_to_load)
|
||||
self._raw = deepcopy(self)
|
||||
self.version = self.get('ver', "0.0.1")
|
||||
if process_now:
|
||||
self.process()
|
||||
|
||||
fee_to_load = deepcopy(fee_dict)
|
||||
def _handle(self, cmd_tpl, value):
|
||||
if cmd_tpl == self.DO_NOTHING:
|
||||
return
|
||||
|
||||
for currency in fee_dict:
|
||||
self._verify_fee(currency, fee_to_load)
|
||||
cmd = cmd_tpl[0]
|
||||
|
||||
if cmd == self.IF_KEY:
|
||||
key, on_key, on_else = cmd_tpl[1:]
|
||||
if key in value:
|
||||
return self._handle(on_key, value)
|
||||
elif on_else:
|
||||
return self._handle(on_else, value)
|
||||
return
|
||||
|
||||
elif cmd == self.IF_VAL:
|
||||
key, v, on_val, on_else = cmd_tpl[1:]
|
||||
if key not in value:
|
||||
return self._handle(on_else, value)
|
||||
if value[key] == v:
|
||||
return self._handle(on_val, value)
|
||||
elif on_else:
|
||||
return self._handle(on_else, value)
|
||||
return
|
||||
|
||||
elif cmd == self.UPDATE:
|
||||
old_key, new_key = cmd_tpl[1:]
|
||||
value.update({new_key: value.pop(old_key)})
|
||||
|
||||
elif cmd == self.REQUIRE:
|
||||
required, validator = cmd_tpl[1:]
|
||||
if required not in self._skip:
|
||||
assert required in value if required not in self else True, "Missing required field: %s, %s" % (required, self.as_json())
|
||||
if required not in self._skip and required in value:
|
||||
self.update({required: value.pop(required)})
|
||||
validator(self[required])
|
||||
else:
|
||||
pass
|
||||
|
||||
elif cmd == self.OPTIONAL:
|
||||
optional, validator = cmd_tpl[1:]
|
||||
if optional in value and optional not in self._skip:
|
||||
self.update({optional: value.pop(optional)})
|
||||
validator(self[optional])
|
||||
else:
|
||||
pass
|
||||
|
||||
elif cmd == self.SKIP:
|
||||
to_skip = cmd_tpl[1]
|
||||
self._skip.append(to_skip)
|
||||
|
||||
elif cmd == self.ADD:
|
||||
key, pushed_val = cmd_tpl[1:]
|
||||
self.update({key: pushed_val})
|
||||
|
||||
elif cmd == self.SUPERSEDE:
|
||||
ver = cmd_tpl[1]
|
||||
self.update({'ver': ver})
|
||||
self.version = ver
|
||||
|
||||
def _load_revision(self, version, value):
|
||||
for k in self.versions[version]:
|
||||
self._handle(k, value)
|
||||
|
||||
def _verify_value(self, value):
|
||||
for version in sorted(self.versions, key=StrictVersion):
|
||||
self._load_revision(version, value)
|
||||
if not value:
|
||||
self['ver'] = version
|
||||
break
|
||||
for skip in self._skip:
|
||||
if skip in value:
|
||||
value.pop(skip)
|
||||
assert value == {}, "Unknown keys: %s, %s" % (json.dumps(value.keys()), self.as_json())
|
||||
|
||||
def _migrate_value(self, value):
|
||||
for migration in self.migrations:
|
||||
self._run_migration(migration, value)
|
||||
|
||||
def _run_migration(self, commands, value):
|
||||
for cmd in commands:
|
||||
self._handle(cmd, value)
|
||||
|
||||
def _run_supersessions(self, value):
|
||||
for cmd in self.supersessions:
|
||||
self._handle(cmd, value)
|
||||
|
||||
|
||||
class LBCFeeValidator(Validator):
|
||||
FV001 = "0.0.1"
|
||||
CURRENT_FEE_VERSION = FV001
|
||||
|
||||
FEE_REVISIONS = {}
|
||||
|
||||
FEE_REVISIONS[FV001] = [
|
||||
(Validator.REQUIRE, 'amount', skip_validate),
|
||||
(Validator.REQUIRE, 'address', skip_validate),
|
||||
]
|
||||
|
||||
FEE_MIGRATIONS = None
|
||||
|
||||
current_version = CURRENT_FEE_VERSION
|
||||
versions = FEE_REVISIONS
|
||||
migrations = FEE_MIGRATIONS
|
||||
|
||||
def __init__(self, fee):
|
||||
Validator.__init__(self, fee)
|
||||
|
||||
|
||||
class BTCFeeValidator(Validator):
|
||||
FV001 = "0.0.1"
|
||||
CURRENT_FEE_VERSION = FV001
|
||||
|
||||
FEE_REVISIONS = {}
|
||||
|
||||
FEE_REVISIONS[FV001] = [
|
||||
(Validator.REQUIRE, 'amount', skip_validate),
|
||||
(Validator.REQUIRE, 'address', skip_validate),
|
||||
]
|
||||
|
||||
FEE_MIGRATIONS = None
|
||||
current_version = CURRENT_FEE_VERSION
|
||||
versions = FEE_REVISIONS
|
||||
migrations = FEE_MIGRATIONS
|
||||
|
||||
def __init__(self, fee):
|
||||
Validator.__init__(self, fee)
|
||||
|
||||
|
||||
class USDFeeValidator(Validator):
|
||||
FV001 = "0.0.1"
|
||||
CURRENT_FEE_VERSION = FV001
|
||||
|
||||
FEE_REVISIONS = {}
|
||||
|
||||
FEE_REVISIONS[FV001] = [
|
||||
(Validator.REQUIRE, 'amount', skip_validate),
|
||||
(Validator.REQUIRE, 'address', skip_validate),
|
||||
]
|
||||
|
||||
FEE_MIGRATIONS = None
|
||||
current_version = CURRENT_FEE_VERSION
|
||||
versions = FEE_REVISIONS
|
||||
migrations = FEE_MIGRATIONS
|
||||
|
||||
def __init__(self, fee):
|
||||
Validator.__init__(self, fee)
|
||||
|
||||
|
||||
class LBRYFeeValidator(Validator):
|
||||
CV001 = "0.0.1"
|
||||
CURRENT_CURRENCY_VERSION = CV001
|
||||
|
||||
CURRENCY_REVISIONS = {}
|
||||
|
||||
CURRENCY_REVISIONS[CV001] = [
|
||||
(Validator.OPTIONAL, 'BTC', BTCFeeValidator),
|
||||
(Validator.OPTIONAL, 'USD', USDFeeValidator),
|
||||
(Validator.OPTIONAL, 'LBC', LBCFeeValidator),
|
||||
]
|
||||
|
||||
CURRENCY_MIGRATIONS = None
|
||||
|
||||
current_version = CURRENT_CURRENCY_VERSION
|
||||
versions = CURRENCY_REVISIONS
|
||||
migrations = CURRENCY_MIGRATIONS
|
||||
|
||||
def __init__(self, fee_dict):
|
||||
Validator.__init__(self, fee_dict)
|
||||
self.currency_symbol = self.keys()[0]
|
||||
self.amount = self._get_amount()
|
||||
self.address = self[self.currency_symbol]['address']
|
||||
|
||||
|
|
@ -65,71 +262,73 @@ class LBRYFeeValidator(dict):
|
|||
log.error('Failed to convert %s to float', amt)
|
||||
raise
|
||||
|
||||
def _verify_fee(self, currency, f):
|
||||
# str in case someone made a claim with a wierd fee
|
||||
assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency)
|
||||
self.currency_symbol = currency
|
||||
self.update({currency: {}})
|
||||
for version in FEE_REVISIONS:
|
||||
self._load_revision(version, f)
|
||||
if not f:
|
||||
self.fee_version = version
|
||||
break
|
||||
assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys())
|
||||
|
||||
def _load_revision(self, version, f):
|
||||
for k in FEE_REVISIONS[version]['required']:
|
||||
assert k in f[self.currency_symbol], "Missing required fee field: %s" % k
|
||||
self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
|
||||
for k in FEE_REVISIONS[version]['optional']:
|
||||
if k in f[self.currency_symbol]:
|
||||
self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
|
||||
class Metadata(Validator):
|
||||
MV001 = "0.0.1"
|
||||
MV002 = "0.0.2"
|
||||
MV003 = "0.0.3"
|
||||
CURRENT_METADATA_VERSION = MV003
|
||||
|
||||
METADATA_REVISIONS = {}
|
||||
|
||||
class Metadata(dict):
|
||||
@classmethod
|
||||
def load_from_hex(cls, metadata):
|
||||
return cls(json.loads(metadata.decode('hex')))
|
||||
METADATA_REVISIONS[MV001] = [
|
||||
(Validator.REQUIRE, 'title', skip_validate),
|
||||
(Validator.REQUIRE, 'description', skip_validate),
|
||||
(Validator.REQUIRE, 'author', skip_validate),
|
||||
(Validator.REQUIRE, 'language', skip_validate),
|
||||
(Validator.REQUIRE, 'license', skip_validate),
|
||||
(Validator.REQUIRE, 'content-type', skip_validate),
|
||||
(Validator.REQUIRE, 'sources', validate_sources),
|
||||
(Validator.OPTIONAL, 'thumbnail', skip_validate),
|
||||
(Validator.OPTIONAL, 'preview', skip_validate),
|
||||
(Validator.OPTIONAL, 'fee', verify_supported_currency),
|
||||
(Validator.OPTIONAL, 'contact', skip_validate),
|
||||
(Validator.OPTIONAL, 'pubkey', skip_validate),
|
||||
]
|
||||
|
||||
METADATA_REVISIONS[MV002] = [
|
||||
(Validator.REQUIRE, 'nsfw', skip_validate),
|
||||
(Validator.REQUIRE, 'ver', skip_validate),
|
||||
(Validator.OPTIONAL, 'license_url', skip_validate),
|
||||
]
|
||||
|
||||
METADATA_REVISIONS[MV003] = [
|
||||
(Validator.REQUIRE, 'content_type', skip_validate),
|
||||
(Validator.SKIP, 'content-type'),
|
||||
(Validator.OPTIONAL, 'sig', skip_validate),
|
||||
(Validator.IF_KEY, 'sig', (Validator.REQUIRE, 'pubkey', skip_validate), Validator.DO_NOTHING),
|
||||
(Validator.IF_KEY, 'pubkey', (Validator.REQUIRE, 'sig', skip_validate), Validator.DO_NOTHING),
|
||||
]
|
||||
|
||||
MIGRATE_MV001_TO_MV002 = [
|
||||
(Validator.IF_KEY, 'nsfw', Validator.DO_NOTHING, (Validator.ADD, 'nsfw', False)),
|
||||
(Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV002)),
|
||||
]
|
||||
MIGRATE_MV002_TO_MV003 = [
|
||||
(Validator.IF_KEY, 'content_type', Validator.DO_NOTHING, (Validator.UPDATE, 'content-type', 'content_type')),
|
||||
(Validator.IF_KEY, 'ver', Validator.DO_NOTHING, (Validator.SUPERSEDE, MV003)),
|
||||
(Validator.IF_VAL, 'ver', MV002, (Validator.SUPERSEDE, MV003), Validator.DO_NOTHING),
|
||||
]
|
||||
|
||||
METADATA_MIGRATIONS = [
|
||||
MIGRATE_MV001_TO_MV002,
|
||||
MIGRATE_MV002_TO_MV003,
|
||||
]
|
||||
|
||||
SUPERSESSIONS = [
|
||||
(Validator.SKIP, 'content-type'),
|
||||
]
|
||||
|
||||
current_version = CURRENT_METADATA_VERSION
|
||||
versions = METADATA_REVISIONS
|
||||
migrations = METADATA_MIGRATIONS
|
||||
supersessions = SUPERSESSIONS
|
||||
|
||||
def __init__(self, metadata):
|
||||
dict.__init__(self)
|
||||
self.meta_version = None
|
||||
metadata_to_load = deepcopy(metadata)
|
||||
Validator.__init__(self, metadata)
|
||||
self.meta_version = self.get('ver', Metadata.MV001)
|
||||
self._load_fee()
|
||||
|
||||
self._verify_sources(metadata_to_load)
|
||||
self._verify_metadata(metadata_to_load)
|
||||
|
||||
def _load_revision(self, version, metadata):
|
||||
for k in METADATA_REVISIONS[version]['required']:
|
||||
assert k in metadata, "Missing required metadata field: %s" % k
|
||||
self.update({k: metadata.pop(k)})
|
||||
for k in METADATA_REVISIONS[version]['optional']:
|
||||
if k == 'fee':
|
||||
self._load_fee(metadata)
|
||||
elif k in metadata:
|
||||
self.update({k: metadata.pop(k)})
|
||||
|
||||
def _load_fee(self, metadata):
|
||||
if 'fee' in metadata:
|
||||
self['fee'] = LBRYFeeValidator(metadata.pop('fee'))
|
||||
|
||||
def _verify_sources(self, metadata):
|
||||
assert "sources" in metadata, "No sources given"
|
||||
for source in metadata['sources']:
|
||||
assert source in SOURCE_TYPES, "Unknown source type"
|
||||
|
||||
def _verify_metadata(self, metadata):
|
||||
for version in METADATA_REVISIONS:
|
||||
self._load_revision(version, metadata)
|
||||
if not metadata:
|
||||
self.meta_version = version
|
||||
if utils.version_is_greater_than(self.meta_version, "0.0.1"):
|
||||
assert self.meta_version == self['ver'], "version mismatch"
|
||||
break
|
||||
assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys())
|
||||
|
||||
def serialize(self):
|
||||
return json.dumps(self).encode("hex")
|
||||
|
||||
def as_json(self):
|
||||
return json.dumps(self)
|
||||
def _load_fee(self):
|
||||
if 'fee' in self:
|
||||
self.update({'fee': LBRYFeeValidator(self['fee'])})
|
||||
|
|
@ -9,7 +9,7 @@ from lbrynet.core.Error import InsufficientFundsError
|
|||
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
|
||||
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION
|
||||
from lbrynet.core.LBRYMetadata import Metadata
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
from lbrynet.conf import LOG_FILE_NAME
|
||||
from twisted.internet import threads, defer
|
||||
|
|
@ -101,9 +101,9 @@ class Publisher(object):
|
|||
return d
|
||||
|
||||
def _claim_name(self):
|
||||
self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory,
|
||||
self.metadata['content_type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory,
|
||||
self.lbry_file.file_name))[0]
|
||||
self.metadata['ver'] = CURRENT_METADATA_VERSION
|
||||
self.metadata['ver'] = Metadata.current_version
|
||||
m = Metadata(self.metadata)
|
||||
|
||||
def set_tx_hash(txid):
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ class MetadataTest(unittest.TestCase):
|
|||
'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg'
|
||||
}
|
||||
m = LBRYMetadata.Metadata(metadata)
|
||||
self.assertFalse('key' in m)
|
||||
self.assertFalse('fee' in m)
|
||||
|
||||
def test_assertion_if_invalid_source(self):
|
||||
metadata = {
|
||||
|
|
|
|||
Loading…
Reference in a new issue