From ca041b5dc46ce26284d85d82e2479ea189635965 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 3 Apr 2017 15:58:20 -0400 Subject: [PATCH] lbryschema integration --- lbrynet/core/Wallet.py | 63 +++++++++++++++-------------- lbrynet/lbrynet_daemon/Daemon.py | 41 +++++++++++++------ lbrynet/lbrynet_daemon/Publisher.py | 36 +++++++---------- tests/unit/core/test_Wallet.py | 18 +++++---- 4 files changed, 86 insertions(+), 72 deletions(-) diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index e195426f0..63bd24fb9 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -8,7 +8,6 @@ from twisted.python.failure import Failure from twisted.enterprise import adbapi from collections import defaultdict, deque from zope.interface import implements -from jsonschema import ValidationError from decimal import Decimal from lbryum import SimpleConfig, Network @@ -16,13 +15,16 @@ from lbryum.lbrycrd import COIN, RECOMMENDED_CLAIMTRIE_HASH_CONFIRMS import lbryum.wallet from lbryum.commands import known_commands, Commands +from lbryschema.claim import ClaimDict +from lbryschema.decode import smart_decode +from lbryschema.error import DecodeError + from lbrynet.core.sqlite_helpers import rerun_if_locked from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet from lbrynet.core.client.ClientRequest import ClientRequest from lbrynet.core.Error import (UnknownNameError, InvalidStreamInfoError, RequestCanceledError, InsufficientFundsError) from lbrynet.db_migrator.migrate1to2 import UNSET_NOUT -from lbrynet.metadata.Metadata import Metadata log = logging.getLogger(__name__) @@ -479,7 +481,8 @@ class Wallet(object): assert k in r, "getvalueforname response missing field %s" % k def _log_success(claim_id): - log.debug("lbry://%s complies with %s, claimid: %s", name, metadata.version, claim_id) + log.debug("lbry://%s complies with %s, claimid: %s", + name, claim_dict.claim_dict['version'], claim_id) return defer.succeed(None) if 'error' in result: @@ -487,15 +490,17 @@ class Wallet(object): return Failure(UnknownNameError(name)) _check_result_fields(result) try: - metadata = Metadata(json.loads(result['value'])) - except (TypeError, ValueError, ValidationError): + claim_dict = smart_decode(result['value'].decode('hex')) + except (TypeError, ValueError, DecodeError): return Failure(InvalidStreamInfoError(name, result['value'])) - sd_hash = metadata['sources']['lbry_sd_hash'] + #TODO: what if keys don't exist here, + # probablly need get_sd_hash() function fro ClaimDict + sd_hash = claim_dict.claim_dict['stream']['source']['source'] claim_outpoint = ClaimOutpoint(result['txid'], result['nout']) d = self._save_name_metadata(name, claim_outpoint, sd_hash) d.addCallback(lambda _: self.get_claimid(name, result['txid'], result['nout'])) d.addCallback(lambda cid: _log_success(cid)) - d.addCallback(lambda _: metadata) + d.addCallback(lambda _: claim_dict.claim_dict) return d def get_claim(self, name, claim_id): @@ -557,7 +562,7 @@ class Wallet(object): d.addErrback(lambda _: False) return d - def _format_claim_for_return(self, name, claim, metadata=None, meta_version=None): + def _format_claim_for_return(self, name, claim, claim_dict, meta_version): result = {} result['claim_id'] = claim['claim_id'] result['amount'] = claim['effective_amount'] @@ -565,28 +570,27 @@ class Wallet(object): result['name'] = name result['txid'] = claim['txid'] result['nout'] = claim['nout'] - result['value'] = metadata if metadata else json.loads(claim['value']) + result['value'] = claim_dict result['supports'] = [ {'txid': support['txid'], 'nout': support['nout']} for support in claim['supports']] - result['meta_version'] = ( - meta_version if meta_version else result['value'].get('ver', '0.0.1')) + result['meta_version'] = meta_version return result def _get_claim_info(self, name, claim_outpoint): def _build_response(claim): try: - metadata = Metadata(json.loads(claim['value'])) - meta_ver = metadata.version - sd_hash = metadata['sources']['lbry_sd_hash'] + claim_dict = smart_decode(claim['value'].decode('hex')) + meta_ver = claim_dict.claim_dict['stream']['metadata']['version'] + sd_hash = claim_dict.claim_dict['stream']['source']['source'] d = self._save_name_metadata(name, claim_outpoint, sd_hash) - except (TypeError, ValueError, ValidationError): - metadata = claim['value'] + except (TypeError, ValueError, KeyError, DecodeError): + claim_dict = claim['value'] meta_ver = "Non-compliant" d = defer.succeed(None) d.addCallback(lambda _: self._format_claim_for_return(name, claim, - metadata=metadata, + claim_dict=claim_dict, meta_version=meta_ver)) log.info( "get claim info lbry://%s metadata: %s, claimid: %s", @@ -622,8 +626,7 @@ class Wallet(object): fee - transaction fee paid to make claim claim_id - claim id of the claim """ - - _metadata = Metadata(metadata) + claim_dict = ClaimDict.load_dict(metadata) my_claim = yield self.get_my_claim(name) if my_claim: @@ -632,13 +635,13 @@ class Wallet(object): raise InsufficientFundsError() old_claim_outpoint = ClaimOutpoint(my_claim['txid'], my_claim['nout']) claim = yield self._send_name_claim_update(name, my_claim['claim_id'], - old_claim_outpoint, _metadata, bid) + old_claim_outpoint, claim_dict.serialized, bid) claim['claim_id'] = my_claim['claim_id'] else: log.info("Making a new claim") if self.get_balance() < bid: raise InsufficientFundsError() - claim = yield self._send_name_claim(name, _metadata, bid) + claim = yield self._send_name_claim(name, claim_dict.serialized, bid) if not claim['success']: msg = 'Claim to name {} failed: {}'.format(name, claim['reason']) @@ -647,8 +650,8 @@ class Wallet(object): claim = self._process_claim_out(claim) claim_outpoint = ClaimOutpoint(claim['txid'], claim['nout']) log.info("Saving metadata for claim %s %d", claim['txid'], claim['nout']) - - yield self._save_name_metadata(name, claim_outpoint, _metadata['sources']['lbry_sd_hash']) + yield self._save_name_metadata(name, claim_outpoint, + claim_dict.claim_dict['stream']['source']['source']) defer.returnValue(claim) @defer.inlineCallbacks @@ -1015,26 +1018,26 @@ class LBRYumWallet(Wallet): return self._run_cmd_as_defer_to_thread('getclaimsforname', name) @defer.inlineCallbacks - def _send_name_claim(self, name, val, amount): + def _send_name_claim(self, name, value, amount): broadcast = False - log.debug("Name claim %s %s %f", name, val, amount) - tx = yield self._run_cmd_as_defer_succeed('claim', name, json.dumps(val), amount, broadcast) + log.debug("Name claim %s %f", name, amount) + tx = yield self._run_cmd_as_defer_succeed('claim', name, value, amount, broadcast) claim_out = yield self._broadcast_claim_transaction(tx) defer.returnValue(claim_out) @defer.inlineCallbacks def _send_name_claim_update(self, name, claim_id, claim_outpoint, value, amount): - metadata = json.dumps(value) - log.debug("Update %s %d %f %s %s '%s'", claim_outpoint['txid'], claim_outpoint['nout'], - amount, name, claim_id, metadata) + log.debug("Update %s %d %f %s %s", claim_outpoint['txid'], claim_outpoint['nout'], + amount, name, claim_id) broadcast = False tx = yield self._run_cmd_as_defer_succeed( 'update', claim_outpoint['txid'], claim_outpoint['nout'], - name, claim_id, metadata, amount, broadcast + name, claim_id, value, amount, broadcast ) claim_out = yield self._broadcast_claim_transaction(tx) defer.returnValue(claim_out) + @defer.inlineCallbacks def _abandon_claim(self, claim_outpoint): log.debug("Abandon %s %s" % (claim_outpoint['txid'], claim_outpoint['nout'])) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index c0bd18f2c..786af1d12 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -710,15 +710,16 @@ class Daemon(AuthJSONRPCServer): defer.returnValue((sd_hash, file_path)) @defer.inlineCallbacks - def _publish_stream(self, name, bid, metadata, file_path=None): + def _publish_stream(self, name, bid, claim_dict, file_path=None): + publisher = Publisher(self.session, self.lbry_file_manager, self.session.wallet) verify_name_characters(name) if bid <= 0.0: raise Exception("Invalid bid") if not file_path: - claim_out = yield publisher.publish_stream(name, bid, metadata) + claim_out = yield publisher.publish_stream(name, bid, claim_dict) else: - claim_out = yield publisher.create_and_publish_stream(name, bid, metadata, file_path) + claim_out = yield publisher.create_and_publish_stream(name, bid, claim_dict, file_path) if conf.settings['reflect_uploads']: d = reupload.reflect_stream(publisher.lbry_file) d.addCallbacks(lambda _: log.info("Reflected new publication to lbry://%s", name), @@ -1672,17 +1673,25 @@ class Daemon(AuthJSONRPCServer): metadata['preview'] = preview if nsfw is not None: metadata['nsfw'] = bool(nsfw) - if sources is not None: - metadata['sources'] = sources - # add address to fee if unspecified + metadata['version'] = '_0_1_0' + + # original format {'currency':{'address','amount'}} + # add address to fee if unspecified {'version': ,'currency', 'address' , 'amount'} if 'fee' in metadata: + new_fee_dict = {} assert len(metadata['fee']) == 1, "Too many fees" - for currency in metadata['fee']: - if 'address' not in metadata['fee'][currency]: - new_address = yield self.session.wallet.get_new_address() - metadata['fee'][currency]['address'] = new_address - metadata['fee'] = FeeValidator(metadata['fee']) + currency, fee_dict = metadata['fee'].items()[0] + if 'address' not in fee_dict: + address = yield self.session.wallet.get_new_address() + else: + address = fee_dict['address'] + new_fee_dict = { + 'version':'_0_0_1', + 'currency': currency, + 'address':address, + 'amount':fee_dict['amount']} + metadata['fee'] = new_fee_dict log.info("Publish: %s", { 'name': name, @@ -1692,7 +1701,15 @@ class Daemon(AuthJSONRPCServer): 'fee': fee, }) - result = yield self._publish_stream(name, bid, metadata, file_path) + claim_dict = { + 'version':'_0_0_1', + 'claimType':'streamType', + 'stream':{'metadata':metadata, 'version':'_0_0_1'}} + + if sources is not None: + claim_dict['stream']['source'] = sources + + result = yield self._publish_stream(name, bid, claim_dict, file_path) response = yield self._render_response(result) defer.returnValue(response) diff --git a/lbrynet/lbrynet_daemon/Publisher.py b/lbrynet/lbrynet_daemon/Publisher.py index ae002dd0a..7f7de4d4c 100644 --- a/lbrynet/lbrynet_daemon/Publisher.py +++ b/lbrynet/lbrynet_daemon/Publisher.py @@ -3,10 +3,10 @@ import mimetypes import os from twisted.internet import defer + from lbrynet.core import file_utils from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob -from lbrynet.metadata.Metadata import Metadata log = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class Publisher(object): Create lbry file and make claim """ @defer.inlineCallbacks - def create_and_publish_stream(self, name, bid, metadata, file_path): + def create_and_publish_stream(self, name, bid, claim_dict, file_path): log.info('Starting publish for %s', name) file_name = os.path.basename(file_path) with file_utils.get_read_handle(file_path) as read_handle: @@ -33,12 +33,14 @@ class Publisher(object): self.lbry_file = yield self.lbry_file_manager.add_lbry_file(stream_hash, prm) sd_hash = yield publish_sd_blob(self.lbry_file_manager.stream_info_manager, self.session.blob_manager, self.lbry_file.stream_hash) - if 'sources' not in metadata: - metadata['sources'] = {} - metadata['sources']['lbry_sd_hash'] = sd_hash - metadata['content_type'] = get_content_type(file_path) - metadata['ver'] = Metadata.current_version - claim_out = yield self.make_claim(name, bid, metadata) + if 'source' not in claim_dict['stream']: + claim_dict['stream']['source'] = {} + claim_dict['stream']['source']['source'] = sd_hash + claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash' + claim_dict['stream']['source']['contentType'] = get_content_type(file_path) + claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here + + claim_out = yield self.make_claim(name, bid, claim_dict) self.lbry_file.completed = True yield self.lbry_file.load_file_attributes() yield self.lbry_file.save_status() @@ -48,23 +50,13 @@ class Publisher(object): Make a claim without creating a lbry file """ @defer.inlineCallbacks - def publish_stream(self, name, bid, metadata): - claim_out = yield self.make_claim(name, bid, metadata) + def publish_stream(self, name, bid, claim_dict): + claim_out = yield self.make_claim(name, bid, claim_dict) defer.returnValue(claim_out) @defer.inlineCallbacks - def update_stream(self, name, bid, metadata): - my_claim = yield self.wallet.get_my_claim(name) - updated_metadata = my_claim['value'] - for meta_key in metadata: - updated_metadata[meta_key] = metadata[meta_key] - claim_out = yield self.make_claim(name, bid, updated_metadata) - defer.returnValue(claim_out) - - @defer.inlineCallbacks - def make_claim(self, name, bid, metadata): - validated_metadata = Metadata(metadata) - claim_out = yield self.wallet.claim_name(name, bid, validated_metadata) + def make_claim(self, name, bid, claim_dict): + claim_out = yield self.wallet.claim_name(name, bid, claim_dict) defer.returnValue(claim_out) diff --git a/tests/unit/core/test_Wallet.py b/tests/unit/core/test_Wallet.py index d6ea02421..b8bbc3048 100644 --- a/tests/unit/core/test_Wallet.py +++ b/tests/unit/core/test_Wallet.py @@ -8,19 +8,21 @@ from lbrynet.core.Wallet import Wallet, ReservedPoints test_metadata = { 'license': 'NASA', -'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, -'ver': '0.0.3', +'version': '_0_1_0', 'description': 'test', 'language': 'en', 'author': 'test', 'title': 'test', -'sources': { - 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, 'nsfw': False, -'content_type': 'video/mp4', 'thumbnail': 'test' } +test_claim_dict = { + 'version':'_0_0_1', + 'claimType':'streamType', + 'stream':{'metadata':test_metadata, 'version':'_0_0_1','source':{'source':'8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb','sourceType':'lbry_sd_hash','contentType':'video/mp4','version':'_0_0_1'}, +}} + class MocLbryumWallet(Wallet): def __init__(self): @@ -42,7 +44,7 @@ class WalletTest(unittest.TestCase): return claim_out MocLbryumWallet._send_name_claim = not_enough_funds_send_name_claim wallet = MocLbryumWallet() - d = wallet.claim_name('test', 1, test_metadata) + d = wallet.claim_name('test', 1, test_claim_dict) self.assertFailure(d,Exception) return d @@ -67,7 +69,7 @@ class WalletTest(unittest.TestCase): MocLbryumWallet._send_name_claim = success_send_name_claim wallet = MocLbryumWallet() - d = wallet.claim_name('test', 1, test_metadata) + d = wallet.claim_name('test', 1, test_claim_dict) d.addCallback(lambda claim_out: check_out(claim_out)) return d @@ -169,7 +171,7 @@ class WalletTest(unittest.TestCase): d = wallet.update_balance() d.addCallback(lambda _: self.assertEqual(5, wallet.get_balance())) d.addCallback(lambda _: wallet.reserve_points('testid',2)) - d.addCallback(lambda _: wallet.claim_name('test', 4, test_metadata)) + d.addCallback(lambda _: wallet.claim_name('test', 4, test_claim_dict)) self.assertFailure(d,InsufficientFundsError) return d