From fd632392d4b65af843624b5e9493dda1301c8b89 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 18 Nov 2019 15:48:52 -0500 Subject: [PATCH] full reposted_claim data returned for claim_search and resolve --- lbry/lbry/extras/daemon/Daemon.py | 27 +++-- .../extras/daemon/json_response_encoder.py | 8 +- lbry/lbry/schema/result.py | 30 +++-- lbry/lbry/schema/types/v2/result_pb2.py | 85 ++++++++------ lbry/lbry/testcase.py | 14 ++- lbry/lbry/wallet/constants.py | 4 +- lbry/lbry/wallet/database.py | 6 +- lbry/lbry/wallet/server/db/reader.py | 106 +++++++++--------- lbry/lbry/wallet/server/db/writer.py | 17 +++ lbry/lbry/wallet/transaction.py | 3 +- lbry/tests/integration/test_claim_commands.py | 62 +++++----- 11 files changed, 210 insertions(+), 152 deletions(-) diff --git a/lbry/lbry/extras/daemon/Daemon.py b/lbry/lbry/extras/daemon/Daemon.py index 7a51de818..d7c208e30 100644 --- a/lbry/lbry/extras/daemon/Daemon.py +++ b/lbry/lbry/extras/daemon/Daemon.py @@ -2061,15 +2061,17 @@ class Daemon(metaclass=JSONRPCServerType): """ @requires(WALLET_COMPONENT) - def jsonrpc_claim_list(self, account_id=None, wallet_id=None, page=None, page_size=None): + def jsonrpc_claim_list(self, claim_type=None, account_id=None, wallet_id=None, page=None, page_size=None): """ List my stream and channel claims. Usage: - claim_list [ | --account_id=] [--wallet_id=] + claim_list [--claim_type=] + [--account_id=] [--wallet_id=] [--page=] [--page_size=] Options: + --claim_type= : (str) claim type: channel, stream, repost, collection --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -2085,7 +2087,7 @@ class Daemon(metaclass=JSONRPCServerType): else: claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts) claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts) - return paginate_rows(claims, claim_count, page, page_size) + return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type) @requires(WALLET_COMPONENT) async def jsonrpc_claim_search(self, **kwargs): @@ -2110,6 +2112,7 @@ class Daemon(metaclass=JSONRPCServerType): [--support_amount=] [--trending_group=] [--trending_mixed=] [--trending_local=] [--trending_global=] [--reposted=] [--claim_type=] [--stream_types=...] [--media_types=...] [--fee_currency=] [--fee_amount=] [--any_tags=...] [--all_tags=...] [--not_tags=...] @@ -2190,6 +2193,9 @@ class Daemon(metaclass=JSONRPCServerType): --trending_global=: (int) trending value calculated relative to all trending content globally (supports equality constraints) + --reposted_claim_id=: (str) all reposts of the specified original claim id + --reposted= : (int) claims reposted this many times (supports + equality constraints) --claim_type= : (str) filter by 'channel', 'stream' or 'unknown' --stream_types= : (list) filter by 'video', 'image', 'document', etc --media_types= : (list) filter by 'video/mp4', 'image/png', etc @@ -2345,8 +2351,9 @@ class Daemon(metaclass=JSONRPCServerType): txo = tx.outputs[0] txo.generate_channel_private_key() + await tx.sign(funding_accounts) + if not preview: - await tx.sign(funding_accounts) account.add_channel_private_key(txo.private_key) wallet.save() await self.broadcast_or_release(tx, blocking) @@ -2500,8 +2507,9 @@ class Daemon(metaclass=JSONRPCServerType): new_txo.script.generate() + await tx.sign(funding_accounts) + if not preview: - await tx.sign(funding_accounts) account.add_channel_private_key(new_txo.private_key) wallet.save() await self.broadcast_or_release(tx, blocking) @@ -2818,8 +2826,7 @@ class Daemon(metaclass=JSONRPCServerType): @requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_repost(self, name, bid, claim_id, allow_duplicate_name=False, channel_id=None, channel_name=None, channel_account_id=None, account_id=None, wallet_id=None, - claim_address=None, funding_account_ids=None, preview=False, blocking=False, - **kwargs): + claim_address=None, funding_account_ids=None, preview=False, blocking=False): """ Creates a claim that references an existing stream by its claim id. @@ -2875,16 +2882,13 @@ class Daemon(metaclass=JSONRPCServerType): ) new_txo = tx.outputs[0] - if not preview: - new_txo.script.generate() - if channel: new_txo.sign(channel) await tx.sign(funding_accounts) if not preview: await self.broadcast_or_release(tx, blocking) - # await self.analytics_manager.send_claim_action('publish') todo: what to send? + await self.analytics_manager.send_claim_action('publish') else: await account.ledger.release_tx(tx) @@ -3459,6 +3463,7 @@ class Daemon(metaclass=JSONRPCServerType): if channel: new_txo.sign(channel) await tx.sign(funding_accounts) + if not preview: await self.broadcast_or_release(tx, blocking) await self.analytics_manager.send_claim_action('publish') diff --git a/lbry/lbry/extras/daemon/json_response_encoder.py b/lbry/lbry/extras/daemon/json_response_encoder.py index 5d8eff677..81054d9c7 100644 --- a/lbry/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/lbry/extras/daemon/json_response_encoder.py @@ -28,14 +28,16 @@ def encode_txo_doc(): 'confirmations': "number of confirmed blocks", 'is_change': "payment to change address, only available when it can be determined", 'is_mine': "payment to one of your accounts, only available when it can be determined", - 'type': "one of 'claim', 'support' or 'payment'", + 'type': "one of 'claim', 'support' or 'purchase'", 'name': "when type is 'claim' or 'support', this is the claim name", - 'claim_id': "when type is 'claim' or 'support', this is the claim id", + 'claim_id': "when type is 'claim', 'support' or 'purchase', this is the claim id", 'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'", 'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload", 'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc", 'protobuf': "hex encoded raw protobuf version of 'value' field", 'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL", + 'claim': "for purchase outputs only, metadata of purchased claim", + 'reposted_claim': "for repost claims only, metadata of claim being reposted", 'signing_channel': "for signed claims only, metadata of signing channel", 'is_channel_signature_valid': "for signed claims only, whether signature is valid", 'purchase_receipt': "metadata for the purchase transaction associated with this claim" @@ -203,6 +205,8 @@ class JSONResponseEncoder(JSONEncoder): output['canonical_url'] = output['meta'].pop('canonical_url') if txo.claims is not None: output['claims'] = [self.encode_output(o) for o in txo.claims] + if txo.reposted_claim is not None: + output['reposted_claim'] = self.encode_output(txo.reposted_claim) if txo.script.is_claim_name or txo.script.is_update_claim: try: output['value'] = txo.claim diff --git a/lbry/lbry/schema/result.py b/lbry/lbry/schema/result.py index f8fef7b29..38cbc372c 100644 --- a/lbry/lbry/schema/result.py +++ b/lbry/lbry/schema/result.py @@ -1,6 +1,6 @@ import base64 import struct -from typing import List +from typing import List, Optional, Tuple from binascii import hexlify from itertools import chain @@ -33,6 +33,7 @@ class Outputs: txo.meta = { 'short_url': f'lbry://{claim.short_url}', 'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}', + 'reposted': claim.reposted, 'is_controlling': claim.is_controlling, 'take_over_height': claim.take_over_height, 'creation_height': claim.creation_height, @@ -47,6 +48,8 @@ class Outputs: } if claim.HasField('channel'): txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout] + if claim.HasField('repost'): + txo.reposted_claim = tx_map[claim.repost.tx_hash].outputs[claim.repost.nout] try: if txo.claim.is_channel: txo.meta['claims_in_channel'] = claim.claims_in_channel @@ -80,13 +83,13 @@ class Outputs: if total is not None: page.total = total for row in txo_rows: - cls.row_to_message(row, page.txos.add()) + cls.row_to_message(row, page.txos.add(), extra_txo_rows) for row in extra_txo_rows: - cls.row_to_message(row, page.extra_txos.add()) + cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows) return page.SerializeToString() @classmethod - def row_to_message(cls, txo, txo_message): + def row_to_message(cls, txo, txo_message, extra_txo_rows): if isinstance(txo, Exception): txo_message.error.text = txo.args[0] if isinstance(txo, ValueError): @@ -98,6 +101,7 @@ class Outputs: txo_message.nout, = struct.unpack(' Transaction: tx = await awaitable if confirm: @@ -245,6 +239,11 @@ class CommandTestCase(IntegrationTestCase): self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm ) + def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs): + return self.confirm_and_render( + self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm + ) + async def stream_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False @@ -307,6 +306,9 @@ class CommandTestCase(IntegrationTestCase): async def file_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items'] + async def claim_list(self, *args, **kwargs): + return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items'] + @staticmethod def get_claim_id(tx): return tx['outputs'][0]['claim_id'] diff --git a/lbry/lbry/wallet/constants.py b/lbry/lbry/wallet/constants.py index fb7e69d0f..52f3454b6 100644 --- a/lbry/lbry/wallet/constants.py +++ b/lbry/lbry/wallet/constants.py @@ -3,11 +3,13 @@ TXO_TYPES = { "channel": 2, "support": 3, "purchase": 4, - "collection": 5 + "collection": 5, + "repost": 6, } CLAIM_TYPES = [ TXO_TYPES['stream'], TXO_TYPES['channel'], TXO_TYPES['collection'], + TXO_TYPES['repost'], ] diff --git a/lbry/lbry/wallet/database.py b/lbry/lbry/wallet/database.py index e8730cf5f..5121e2af9 100644 --- a/lbry/lbry/wallet/database.py +++ b/lbry/lbry/wallet/database.py @@ -140,7 +140,11 @@ class WalletDatabase(BaseDatabase): @staticmethod def constrain_claims(constraints): - constraints['txo_type__in'] = CLAIM_TYPES + claim_type = constraints.pop('claim_type', None) + if claim_type is not None: + constraints['txo_type'] = TXO_TYPES[claim_type] + else: + constraints['txo_type__in'] = CLAIM_TYPES async def get_claims(self, **constraints) -> List[Output]: self.constrain_claims(constraints) diff --git a/lbry/lbry/wallet/server/db/reader.py b/lbry/lbry/wallet/server/db/reader.py index 7fb13fdb8..3320cda7f 100644 --- a/lbry/lbry/wallet/server/db/reader.py +++ b/lbry/lbry/wallet/server/db/reader.py @@ -39,7 +39,7 @@ ATTRIBUTE_ARRAY_MAX_LENGTH = 100 INTEGER_PARAMS = { 'height', 'creation_height', 'activation_height', 'expiration_height', 'timestamp', 'creation_timestamp', 'release_time', 'fee_amount', - 'tx_position', 'channel_join', + 'tx_position', 'channel_join', 'reposted', 'amount', 'effective_amount', 'support_amount', 'trending_group', 'trending_mixed', 'trending_local', 'trending_global', @@ -267,12 +267,13 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]: sqlite3.Binary(unhexlify(channel_id)[::-1]) for channel_id in blocklist_ids ] constraints.update({ - f'$blocking_channels{i}': a for i, a in enumerate(blocking_channels) + f'$blocking_channel{i}': a for i, a in enumerate(blocking_channels) }) - blocklist = ', '.join([f':$blocking_channels{i}' for i in range(len(blocking_channels))]) - constraints['claim.claim_hash__not_in'] = f""" - SELECT reposted_claim_hash FROM claim - WHERE channel_hash IN ({blocklist}) + blocklist = ', '.join([ + f':$blocking_channel{i}' for i in range(len(blocking_channels)) + ]) + constraints['claim.claim_hash__not_in#blocklist_channel_ids'] = f""" + SELECT reposted_claim_hash FROM claim WHERE channel_hash IN ({blocklist}) """ if 'signature_valid' in constraints: has_channel_signature = constraints.pop('has_channel_signature', False) @@ -319,14 +320,9 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]: select = f"SELECT {cols} FROM search JOIN claim ON (search.rowid=claim.rowid)" else: select = f"SELECT {cols} FROM claim" - - sql, values = query( - select if for_count else select+""" - LEFT JOIN claimtrie USING (claim_hash) - LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash) - """, **constraints - ) - return sql, values + if not for_count: + select += " LEFT JOIN claimtrie USING (claim_hash)" + return query(select, **constraints) def get_claims(cols, for_count=False, **constraints) -> List: @@ -350,6 +346,46 @@ def get_claims_count(**constraints) -> int: return count[0][0] +def _search(**constraints): + return get_claims( + """ + claimtrie.claim_hash as is_controlling, + claimtrie.last_take_over_height, + claim.claim_hash, claim.txo_hash, + claim.claims_in_channel, claim.reposted, + claim.height, claim.creation_height, + claim.activation_height, claim.expiration_height, + claim.effective_amount, claim.support_amount, + claim.trending_group, claim.trending_mixed, + claim.trending_local, claim.trending_global, + claim.short_url, claim.canonical_url, + claim.channel_hash, claim.reposted_claim_hash, + claim.signature_valid + """, **constraints + ) + + +def _get_referenced_rows(txo_rows: List[sqlite3.Row]): + repost_hashes = set(filter(None, map(itemgetter('reposted_claim_hash'), txo_rows))) + channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows))) + + reposted_txos = [] + if repost_hashes: + reposted_txos = _search( + **{'claim.claim_hash__in': [sqlite3.Binary(h) for h in repost_hashes]} + ) + channel_hashes |= set(filter(None, map(itemgetter('channel_hash'), reposted_txos))) + + channel_txos = [] + if channel_hashes: + channel_txos = _search( + **{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]} + ) + + # channels must come first for client side inflation to work properly + return channel_txos + reposted_txos + + @measure def search(constraints) -> Tuple[List, List, int, int]: assert set(constraints).issubset(SEARCH_PARAMS), \ @@ -362,49 +398,15 @@ def search(constraints) -> Tuple[List, List, int, int]: if 'order_by' not in constraints: constraints['order_by'] = ["claim_hash"] txo_rows = _search(**constraints) - channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows))) - extra_txo_rows = [] - if channel_hashes: - extra_txo_rows = _search( - **{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]} - ) + extra_txo_rows = _get_referenced_rows(txo_rows) return txo_rows, extra_txo_rows, constraints['offset'], total -def _search(**constraints): - return get_claims( - """ - claimtrie.claim_hash as is_controlling, - claimtrie.last_take_over_height, - claim.claim_hash, claim.txo_hash, - claim.claims_in_channel, - claim.height, claim.creation_height, - claim.activation_height, claim.expiration_height, - claim.effective_amount, claim.support_amount, - claim.trending_group, claim.trending_mixed, - claim.trending_local, claim.trending_global, - claim.short_url, claim.canonical_url, claim.reposted_claim_hash, - claim.channel_hash, channel.txo_hash AS channel_txo_hash, - channel.height AS channel_height, claim.signature_valid - """, **constraints - ) - - @measure def resolve(urls) -> Tuple[List, List]: - result = [] - channel_hashes = set() - for raw_url in urls: - match = resolve_url(raw_url) - result.append(match) - if isinstance(match, sqlite3.Row) and match['channel_hash']: - channel_hashes.add(match['channel_hash']) - extra_txo_rows = [] - if channel_hashes: - extra_txo_rows = _search( - **{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]} - ) - return result, extra_txo_rows + txo_rows = [resolve_url(raw_url) for raw_url in urls] + extra_txo_rows = _get_referenced_rows([r for r in txo_rows if isinstance(r, sqlite3.Row)]) + return txo_rows, extra_txo_rows @measure diff --git a/lbry/lbry/wallet/server/db/writer.py b/lbry/lbry/wallet/server/db/writer.py index aa46a9de6..ba99c4ff1 100644 --- a/lbry/lbry/wallet/server/db/writer.py +++ b/lbry/lbry/wallet/server/db/writer.py @@ -55,6 +55,7 @@ class SQLDB: description text, claim_type integer, + reposted integer default 0, -- streams stream_type text, @@ -385,6 +386,21 @@ class SQLDB: 'support', {'txo_hash__in': [sqlite3.Binary(txo_hash) for txo_hash in txo_hashes]} )) + def calculate_reposts(self, claims: List[Output]): + targets = set() + for claim in claims: + if claim.claim.is_repost: + targets.add((claim.claim.repost.reference.claim_hash,)) + if targets: + self.db.executemany( + """ + UPDATE claim SET reposted = ( + SELECT count(*) FROM claim AS repost WHERE repost.reposted_claim_hash = claim.claim_hash + ) + WHERE claim_hash = ? + """, targets + ) + def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims, affected_channels, timer): if not new_claims and not updated_claims and not spent_claims: return @@ -716,6 +732,7 @@ class SQLDB: affected_channels = r(self.delete_claims, delete_claim_hashes) r(self.delete_supports, delete_support_txo_hashes) r(self.insert_claims, insert_claims, header) + r(self.calculate_reposts, insert_claims) r(update_full_text_search, 'after-insert', [txo.claim_hash for txo in insert_claims], self.db, height, daemon_height, self.main.first_sync) r(update_full_text_search, 'before-update', diff --git a/lbry/lbry/wallet/transaction.py b/lbry/lbry/wallet/transaction.py index 4de21de7b..819271239 100644 --- a/lbry/lbry/wallet/transaction.py +++ b/lbry/lbry/wallet/transaction.py @@ -32,7 +32,7 @@ class Output(BaseOutput): __slots__ = ( 'channel', 'private_key', 'meta', 'purchase', 'purchased_claim', 'purchase_receipt', - 'claims', + 'reposted_claim', 'claims', ) def __init__(self, *args, channel: Optional['Output'] = None, @@ -43,6 +43,7 @@ class Output(BaseOutput): self.purchase: 'Output' = None # txo containing purchase metadata self.purchased_claim: 'Output' = None # resolved claim pointed to by purchase self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim + self.reposted_claim: 'Output' = None # txo representing claim being reposted self.claims: List['Output'] = None # resolved claims for collection self.meta = {} diff --git a/lbry/tests/integration/test_claim_commands.py b/lbry/tests/integration/test_claim_commands.py index 4526c6bbf..59f7c1bf1 100644 --- a/lbry/tests/integration/test_claim_commands.py +++ b/lbry/tests/integration/test_claim_commands.py @@ -717,44 +717,38 @@ class StreamCommands(ClaimTestCase): async def test_repost(self): await self.channel_create('@goodies', '1.0') tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies') - claim_id = tx['outputs'][0]['claim_id'] - await self.stream_repost(claim_id, 'newstuff-again', '1.1') - claim_list = (await self.out(self.daemon.jsonrpc_claim_list()))['items'] - reposts_on_claim_list = [claim for claim in claim_list if claim['value_type'] == 'repost'] - self.assertEqual(len(reposts_on_claim_list), 1) + claim_id = self.get_claim_id(tx) + + self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0) + + tx = await self.stream_repost(claim_id, 'newstuff-again', '1.1') + repost_id = self.get_claim_id(tx) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 1) + self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1) + self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id) + await self.channel_create('@reposting-goodies', '1.0') await self.stream_repost(claim_id, 'repost-on-channel', '1.1', channel_name='@reposting-goodies') - claim_list = (await self.out(self.daemon.jsonrpc_claim_list()))['items'] - reposts_on_claim_list = [claim for claim in claim_list if claim['value_type'] == 'repost'] - self.assertEqual(len(reposts_on_claim_list), 2) - signed_reposts = [repost for repost in reposts_on_claim_list if repost.get('is_channel_signature_valid')] - self.assertEqual(len(signed_reposts), 1) - # check that its directly searchable (simplest case, by name) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 2) + self.assertItemCount(await self.daemon.jsonrpc_claim_search(reposted_claim_id=claim_id), 2) + self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 2) + + search_results = await self.claim_search(reposted='>=2') + self.assertEqual(len(search_results), 1) + self.assertEqual(search_results[0]['name'], 'newstuff') + search_results = await self.claim_search(name='repost-on-channel') self.assertEqual(len(search_results), 1) - self.assertTrue( - any(claim['claim_id'] for claim in reposts_on_claim_list - if claim['name'] == 'repost-on-channel' and claim['claim_id'] == search_results[0]['claim_id']) - ) - search_results = await self.claim_search(name='newstuff-again') - self.assertEqual(len(search_results), 1) - self.assertTrue( - any(claim['claim_id'] for claim in reposts_on_claim_list - if claim['name'] == 'newstuff-again' and claim['claim_id'] == search_results[0]['claim_id']) - ) - # complex case, reverse search (reposts for claim id) - reposts = await self.claim_search(reposted_claim_id=claim_id) - self.assertEqual(len(reposts), 2) - self.assertSetEqual( - {repost['claim_id'] for repost in reposts}, - {claim['claim_id'] for claim in reposts_on_claim_list} - ) - # check that it resolves fine too - resolved_reposts = await self.resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again']) - self.assertEqual( - [resolution['claim_id'] for resolution in resolved_reposts.values()], - [claim['claim_id'] for claim in reposts_on_claim_list] - ) + search = search_results[0] + self.assertEqual(search['name'], 'repost-on-channel') + self.assertEqual(search['signing_channel']['name'], '@reposting-goodies') + self.assertEqual(search['reposted_claim']['name'], 'newstuff') + self.assertEqual(search['reposted_claim']['meta']['reposted'], 2) + self.assertEqual(search['reposted_claim']['signing_channel']['name'], '@goodies') + + resolved = await self.resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again']) + self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search) + self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff') async def test_filtering_channels_for_removing_content(self): await self.channel_create('@badstuff', '1.0')