From ca041b5dc46ce26284d85d82e2479ea189635965 Mon Sep 17 00:00:00 2001
From: Kay Kurokawa <kay@lbry.io>
Date: Mon, 3 Apr 2017 15:58:20 -0400
Subject: [PATCH] lbryschema integration
---
lbrynet/core/Wallet.py | 63 +++++++++++++++--------------
lbrynet/lbrynet_daemon/Daemon.py | 41 +++++++++++++------
lbrynet/lbrynet_daemon/Publisher.py | 36 +++++++----------
tests/unit/core/test_Wallet.py | 18 +++++----
4 files changed, 86 insertions(+), 72 deletions(-)
diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py
index e195426f0..63bd24fb9 100644
--- a/lbrynet/core/Wallet.py
+++ b/lbrynet/core/Wallet.py
@@ -8,7 +8,6 @@ from twisted.python.failure import Failure
from twisted.enterprise import adbapi
from collections import defaultdict, deque
from zope.interface import implements
-from jsonschema import ValidationError
from decimal import Decimal
from lbryum import SimpleConfig, Network
@@ -16,13 +15,16 @@ from lbryum.lbrycrd import COIN, RECOMMENDED_CLAIMTRIE_HASH_CONFIRMS
import lbryum.wallet
from lbryum.commands import known_commands, Commands
+from lbryschema.claim import ClaimDict
+from lbryschema.decode import smart_decode
+from lbryschema.error import DecodeError
+
from lbrynet.core.sqlite_helpers import rerun_if_locked
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet
from lbrynet.core.client.ClientRequest import ClientRequest
from lbrynet.core.Error import (UnknownNameError, InvalidStreamInfoError, RequestCanceledError,
InsufficientFundsError)
from lbrynet.db_migrator.migrate1to2 import UNSET_NOUT
-from lbrynet.metadata.Metadata import Metadata
log = logging.getLogger(__name__)
@@ -479,7 +481,8 @@ class Wallet(object):
assert k in r, "getvalueforname response missing field %s" % k
def _log_success(claim_id):
- log.debug("lbry://%s complies with %s, claimid: %s", name, metadata.version, claim_id)
+ log.debug("lbry://%s complies with %s, claimid: %s",
+ name, claim_dict.claim_dict['version'], claim_id)
return defer.succeed(None)
if 'error' in result:
@@ -487,15 +490,17 @@ class Wallet(object):
return Failure(UnknownNameError(name))
_check_result_fields(result)
try:
- metadata = Metadata(json.loads(result['value']))
- except (TypeError, ValueError, ValidationError):
+ claim_dict = smart_decode(result['value'].decode('hex'))
+ except (TypeError, ValueError, DecodeError):
return Failure(InvalidStreamInfoError(name, result['value']))
- sd_hash = metadata['sources']['lbry_sd_hash']
+ #TODO: what if keys don't exist here,
+ # probablly need get_sd_hash() function fro ClaimDict
+ sd_hash = claim_dict.claim_dict['stream']['source']['source']
claim_outpoint = ClaimOutpoint(result['txid'], result['nout'])
d = self._save_name_metadata(name, claim_outpoint, sd_hash)
d.addCallback(lambda _: self.get_claimid(name, result['txid'], result['nout']))
d.addCallback(lambda cid: _log_success(cid))
- d.addCallback(lambda _: metadata)
+ d.addCallback(lambda _: claim_dict.claim_dict)
return d
def get_claim(self, name, claim_id):
@@ -557,7 +562,7 @@ class Wallet(object):
d.addErrback(lambda _: False)
return d
- def _format_claim_for_return(self, name, claim, metadata=None, meta_version=None):
+ def _format_claim_for_return(self, name, claim, claim_dict, meta_version):
result = {}
result['claim_id'] = claim['claim_id']
result['amount'] = claim['effective_amount']
@@ -565,28 +570,27 @@ class Wallet(object):
result['name'] = name
result['txid'] = claim['txid']
result['nout'] = claim['nout']
- result['value'] = metadata if metadata else json.loads(claim['value'])
+ result['value'] = claim_dict
result['supports'] = [
{'txid': support['txid'], 'nout': support['nout']} for support in claim['supports']]
- result['meta_version'] = (
- meta_version if meta_version else result['value'].get('ver', '0.0.1'))
+ result['meta_version'] = meta_version
return result
def _get_claim_info(self, name, claim_outpoint):
def _build_response(claim):
try:
- metadata = Metadata(json.loads(claim['value']))
- meta_ver = metadata.version
- sd_hash = metadata['sources']['lbry_sd_hash']
+ claim_dict = smart_decode(claim['value'].decode('hex'))
+ meta_ver = claim_dict.claim_dict['stream']['metadata']['version']
+ sd_hash = claim_dict.claim_dict['stream']['source']['source']
d = self._save_name_metadata(name, claim_outpoint, sd_hash)
- except (TypeError, ValueError, ValidationError):
- metadata = claim['value']
+ except (TypeError, ValueError, KeyError, DecodeError):
+ claim_dict = claim['value']
meta_ver = "Non-compliant"
d = defer.succeed(None)
d.addCallback(lambda _: self._format_claim_for_return(name,
claim,
- metadata=metadata,
+ claim_dict=claim_dict,
meta_version=meta_ver))
log.info(
"get claim info lbry://%s metadata: %s, claimid: %s",
@@ -622,8 +626,7 @@ class Wallet(object):
fee - transaction fee paid to make claim
claim_id - claim id of the claim
"""
-
- _metadata = Metadata(metadata)
+ claim_dict = ClaimDict.load_dict(metadata)
my_claim = yield self.get_my_claim(name)
if my_claim:
@@ -632,13 +635,13 @@ class Wallet(object):
raise InsufficientFundsError()
old_claim_outpoint = ClaimOutpoint(my_claim['txid'], my_claim['nout'])
claim = yield self._send_name_claim_update(name, my_claim['claim_id'],
- old_claim_outpoint, _metadata, bid)
+ old_claim_outpoint, claim_dict.serialized, bid)
claim['claim_id'] = my_claim['claim_id']
else:
log.info("Making a new claim")
if self.get_balance() < bid:
raise InsufficientFundsError()
- claim = yield self._send_name_claim(name, _metadata, bid)
+ claim = yield self._send_name_claim(name, claim_dict.serialized, bid)
if not claim['success']:
msg = 'Claim to name {} failed: {}'.format(name, claim['reason'])
@@ -647,8 +650,8 @@ class Wallet(object):
claim = self._process_claim_out(claim)
claim_outpoint = ClaimOutpoint(claim['txid'], claim['nout'])
log.info("Saving metadata for claim %s %d", claim['txid'], claim['nout'])
-
- yield self._save_name_metadata(name, claim_outpoint, _metadata['sources']['lbry_sd_hash'])
+ yield self._save_name_metadata(name, claim_outpoint,
+ claim_dict.claim_dict['stream']['source']['source'])
defer.returnValue(claim)
@defer.inlineCallbacks
@@ -1015,26 +1018,26 @@ class LBRYumWallet(Wallet):
return self._run_cmd_as_defer_to_thread('getclaimsforname', name)
@defer.inlineCallbacks
- def _send_name_claim(self, name, val, amount):
+ def _send_name_claim(self, name, value, amount):
broadcast = False
- log.debug("Name claim %s %s %f", name, val, amount)
- tx = yield self._run_cmd_as_defer_succeed('claim', name, json.dumps(val), amount, broadcast)
+ log.debug("Name claim %s %f", name, amount)
+ tx = yield self._run_cmd_as_defer_succeed('claim', name, value, amount, broadcast)
claim_out = yield self._broadcast_claim_transaction(tx)
defer.returnValue(claim_out)
@defer.inlineCallbacks
def _send_name_claim_update(self, name, claim_id, claim_outpoint, value, amount):
- metadata = json.dumps(value)
- log.debug("Update %s %d %f %s %s '%s'", claim_outpoint['txid'], claim_outpoint['nout'],
- amount, name, claim_id, metadata)
+ log.debug("Update %s %d %f %s %s", claim_outpoint['txid'], claim_outpoint['nout'],
+ amount, name, claim_id)
broadcast = False
tx = yield self._run_cmd_as_defer_succeed(
'update', claim_outpoint['txid'], claim_outpoint['nout'],
- name, claim_id, metadata, amount, broadcast
+ name, claim_id, value, amount, broadcast
)
claim_out = yield self._broadcast_claim_transaction(tx)
defer.returnValue(claim_out)
+
@defer.inlineCallbacks
def _abandon_claim(self, claim_outpoint):
log.debug("Abandon %s %s" % (claim_outpoint['txid'], claim_outpoint['nout']))
diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py
index c0bd18f2c..786af1d12 100644
--- a/lbrynet/lbrynet_daemon/Daemon.py
+++ b/lbrynet/lbrynet_daemon/Daemon.py
@@ -710,15 +710,16 @@ class Daemon(AuthJSONRPCServer):
defer.returnValue((sd_hash, file_path))
@defer.inlineCallbacks
- def _publish_stream(self, name, bid, metadata, file_path=None):
+ def _publish_stream(self, name, bid, claim_dict, file_path=None):
+
publisher = Publisher(self.session, self.lbry_file_manager, self.session.wallet)
verify_name_characters(name)
if bid <= 0.0:
raise Exception("Invalid bid")
if not file_path:
- claim_out = yield publisher.publish_stream(name, bid, metadata)
+ claim_out = yield publisher.publish_stream(name, bid, claim_dict)
else:
- claim_out = yield publisher.create_and_publish_stream(name, bid, metadata, file_path)
+ claim_out = yield publisher.create_and_publish_stream(name, bid, claim_dict, file_path)
if conf.settings['reflect_uploads']:
d = reupload.reflect_stream(publisher.lbry_file)
d.addCallbacks(lambda _: log.info("Reflected new publication to lbry://%s", name),
@@ -1672,17 +1673,25 @@ class Daemon(AuthJSONRPCServer):
metadata['preview'] = preview
if nsfw is not None:
metadata['nsfw'] = bool(nsfw)
- if sources is not None:
- metadata['sources'] = sources
- # add address to fee if unspecified
+ metadata['version'] = '_0_1_0'
+
+ # original format {'currency':{'address','amount'}}
+ # add address to fee if unspecified {'version': ,'currency', 'address' , 'amount'}
if 'fee' in metadata:
+ new_fee_dict = {}
assert len(metadata['fee']) == 1, "Too many fees"
- for currency in metadata['fee']:
- if 'address' not in metadata['fee'][currency]:
- new_address = yield self.session.wallet.get_new_address()
- metadata['fee'][currency]['address'] = new_address
- metadata['fee'] = FeeValidator(metadata['fee'])
+ currency, fee_dict = metadata['fee'].items()[0]
+ if 'address' not in fee_dict:
+ address = yield self.session.wallet.get_new_address()
+ else:
+ address = fee_dict['address']
+ new_fee_dict = {
+ 'version':'_0_0_1',
+ 'currency': currency,
+ 'address':address,
+ 'amount':fee_dict['amount']}
+ metadata['fee'] = new_fee_dict
log.info("Publish: %s", {
'name': name,
@@ -1692,7 +1701,15 @@ class Daemon(AuthJSONRPCServer):
'fee': fee,
})
- result = yield self._publish_stream(name, bid, metadata, file_path)
+ claim_dict = {
+ 'version':'_0_0_1',
+ 'claimType':'streamType',
+ 'stream':{'metadata':metadata, 'version':'_0_0_1'}}
+
+ if sources is not None:
+ claim_dict['stream']['source'] = sources
+
+ result = yield self._publish_stream(name, bid, claim_dict, file_path)
response = yield self._render_response(result)
defer.returnValue(response)
diff --git a/lbrynet/lbrynet_daemon/Publisher.py b/lbrynet/lbrynet_daemon/Publisher.py
index ae002dd0a..7f7de4d4c 100644
--- a/lbrynet/lbrynet_daemon/Publisher.py
+++ b/lbrynet/lbrynet_daemon/Publisher.py
@@ -3,10 +3,10 @@ import mimetypes
import os
from twisted.internet import defer
+
from lbrynet.core import file_utils
from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
-from lbrynet.metadata.Metadata import Metadata
log = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ class Publisher(object):
Create lbry file and make claim
"""
@defer.inlineCallbacks
- def create_and_publish_stream(self, name, bid, metadata, file_path):
+ def create_and_publish_stream(self, name, bid, claim_dict, file_path):
log.info('Starting publish for %s', name)
file_name = os.path.basename(file_path)
with file_utils.get_read_handle(file_path) as read_handle:
@@ -33,12 +33,14 @@ class Publisher(object):
self.lbry_file = yield self.lbry_file_manager.add_lbry_file(stream_hash, prm)
sd_hash = yield publish_sd_blob(self.lbry_file_manager.stream_info_manager,
self.session.blob_manager, self.lbry_file.stream_hash)
- if 'sources' not in metadata:
- metadata['sources'] = {}
- metadata['sources']['lbry_sd_hash'] = sd_hash
- metadata['content_type'] = get_content_type(file_path)
- metadata['ver'] = Metadata.current_version
- claim_out = yield self.make_claim(name, bid, metadata)
+ if 'source' not in claim_dict['stream']:
+ claim_dict['stream']['source'] = {}
+ claim_dict['stream']['source']['source'] = sd_hash
+ claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
+ claim_dict['stream']['source']['contentType'] = get_content_type(file_path)
+ claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
+
+ claim_out = yield self.make_claim(name, bid, claim_dict)
self.lbry_file.completed = True
yield self.lbry_file.load_file_attributes()
yield self.lbry_file.save_status()
@@ -48,23 +50,13 @@ class Publisher(object):
Make a claim without creating a lbry file
"""
@defer.inlineCallbacks
- def publish_stream(self, name, bid, metadata):
- claim_out = yield self.make_claim(name, bid, metadata)
+ def publish_stream(self, name, bid, claim_dict):
+ claim_out = yield self.make_claim(name, bid, claim_dict)
defer.returnValue(claim_out)
@defer.inlineCallbacks
- def update_stream(self, name, bid, metadata):
- my_claim = yield self.wallet.get_my_claim(name)
- updated_metadata = my_claim['value']
- for meta_key in metadata:
- updated_metadata[meta_key] = metadata[meta_key]
- claim_out = yield self.make_claim(name, bid, updated_metadata)
- defer.returnValue(claim_out)
-
- @defer.inlineCallbacks
- def make_claim(self, name, bid, metadata):
- validated_metadata = Metadata(metadata)
- claim_out = yield self.wallet.claim_name(name, bid, validated_metadata)
+ def make_claim(self, name, bid, claim_dict):
+ claim_out = yield self.wallet.claim_name(name, bid, claim_dict)
defer.returnValue(claim_out)
diff --git a/tests/unit/core/test_Wallet.py b/tests/unit/core/test_Wallet.py
index d6ea02421..b8bbc3048 100644
--- a/tests/unit/core/test_Wallet.py
+++ b/tests/unit/core/test_Wallet.py
@@ -8,19 +8,21 @@ from lbrynet.core.Wallet import Wallet, ReservedPoints
test_metadata = {
'license': 'NASA',
-'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}},
-'ver': '0.0.3',
+'version': '_0_1_0',
'description': 'test',
'language': 'en',
'author': 'test',
'title': 'test',
-'sources': {
- 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'},
'nsfw': False,
-'content_type': 'video/mp4',
'thumbnail': 'test'
}
+test_claim_dict = {
+ 'version':'_0_0_1',
+ 'claimType':'streamType',
+ 'stream':{'metadata':test_metadata, 'version':'_0_0_1','source':{'source':'8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb','sourceType':'lbry_sd_hash','contentType':'video/mp4','version':'_0_0_1'},
+}}
+
class MocLbryumWallet(Wallet):
def __init__(self):
@@ -42,7 +44,7 @@ class WalletTest(unittest.TestCase):
return claim_out
MocLbryumWallet._send_name_claim = not_enough_funds_send_name_claim
wallet = MocLbryumWallet()
- d = wallet.claim_name('test', 1, test_metadata)
+ d = wallet.claim_name('test', 1, test_claim_dict)
self.assertFailure(d,Exception)
return d
@@ -67,7 +69,7 @@ class WalletTest(unittest.TestCase):
MocLbryumWallet._send_name_claim = success_send_name_claim
wallet = MocLbryumWallet()
- d = wallet.claim_name('test', 1, test_metadata)
+ d = wallet.claim_name('test', 1, test_claim_dict)
d.addCallback(lambda claim_out: check_out(claim_out))
return d
@@ -169,7 +171,7 @@ class WalletTest(unittest.TestCase):
d = wallet.update_balance()
d.addCallback(lambda _: self.assertEqual(5, wallet.get_balance()))
d.addCallback(lambda _: wallet.reserve_points('testid',2))
- d.addCallback(lambda _: wallet.claim_name('test', 4, test_metadata))
+ d.addCallback(lambda _: wallet.claim_name('test', 4, test_claim_dict))
self.assertFailure(d,InsufficientFundsError)
return d