From 3ff9e99416c69340b8d109139250e2a2d68135a7 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sat, 7 Mar 2020 00:34:47 -0500 Subject: [PATCH 1/2] added txo_list command --- lbry/extras/daemon/daemon.py | 139 +++++++++++------- lbry/testcase.py | 6 + lbry/wallet/account.py | 6 + lbry/wallet/constants.py | 15 +- lbry/wallet/database.py | 65 +++++--- lbry/wallet/ledger.py | 15 +- .../blockchain/test_claim_commands.py | 132 +++++++++++++++-- 7 files changed, 280 insertions(+), 98 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 2a26f0544..1e796e4d7 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -21,9 +21,10 @@ from prometheus_client import generate_latest as prom_generate_latest from google.protobuf.message import DecodeError from lbry.wallet import ( Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic, - Transaction, Output, Input, Account + Transaction, Output, Input, Account, database ) from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc +from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES from lbry import utils from lbry.conf import Config, Setting, NOT_SET @@ -2167,19 +2168,20 @@ class Daemon(metaclass=JSONRPCServerType): """ @requires(WALLET_COMPONENT) - def jsonrpc_claim_list( - self, claim_type=None, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False): + def jsonrpc_claim_list(self, claim_type=None, **kwargs): """ List my stream and channel claims. Usage: - claim_list [--claim_type=...] + claim_list [--claim_type=...] [--claim_id=...] [--name=...] [--account_id=] [--wallet_id=] [--page=] [--page_size=] [--resolve] Options: - --claim_type= : (str) claim type: channel, stream, repost, collection + --claim_type= : (str or list) claim type: channel, stream, repost, collection + --claim_id= : (str or list) claim id + --name= : (str or list) claim name --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -2188,15 +2190,9 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {Paginated[Output]} """ - wallet = self.wallet_manager.get_wallet_or_default(wallet_id) - if account_id: - account = wallet.get_account_or_error(account_id) - claims = account.get_claims - claim_count = account.get_claim_count - else: - claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts) - claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts) - return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type, resolve=resolve) + kwargs['type'] = claim_type or CLAIM_TYPE_NAMES + kwargs['unspent'] = True + return self.jsonrpc_txo_list(**kwargs) @requires(WALLET_COMPONENT) async def jsonrpc_claim_search(self, **kwargs): @@ -2699,15 +2695,18 @@ class Daemon(metaclass=JSONRPCServerType): return tx @requires(WALLET_COMPONENT) - def jsonrpc_channel_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False): + def jsonrpc_channel_list(self, *args, **kwargs): """ List my channel claims. Usage: channel_list [ | --account_id=] [--wallet_id=] + [--name=...] [--claim_id=...] [--page=] [--page_size=] [--resolve] Options: + --name= : (str or list) channel name + --claim_id= : (str or list) channel id --account_id= : (str) id of the account to use --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -2716,15 +2715,9 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {Paginated[Output]} """ - wallet = self.wallet_manager.get_wallet_or_default(wallet_id) - if account_id: - account = wallet.get_account_or_error(account_id) - channels = account.get_channels - channel_count = account.get_channel_count - else: - channels = partial(self.ledger.get_channels, wallet=wallet, accounts=wallet.accounts) - channel_count = partial(self.ledger.get_channel_count, wallet=wallet, accounts=wallet.accounts) - return paginate_rows(channels, channel_count, page, page_size, resolve=resolve) + kwargs['type'] = 'channel' + kwargs['unspent'] = True + return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) async def jsonrpc_channel_export(self, channel_id=None, channel_name=None, account_id=None, wallet_id=None): @@ -3441,15 +3434,18 @@ class Daemon(metaclass=JSONRPCServerType): return tx @requires(WALLET_COMPONENT) - def jsonrpc_stream_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False): + def jsonrpc_stream_list(self, *args, **kwargs): """ List my stream claims. Usage: stream_list [ | --account_id=] [--wallet_id=] - [--page=] [--page_size=] [--resolve] + [--name=...] [--claim_id=...] + [--page=] [--page_size=] [--resolve] Options: + --name= : (str or list) stream name + --claim_id= : (str or list) stream id --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -3458,15 +3454,9 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {Paginated[Output]} """ - wallet = self.wallet_manager.get_wallet_or_default(wallet_id) - if account_id: - account = wallet.get_account_or_error(account_id) - streams = account.get_streams - stream_count = account.get_stream_count - else: - streams = partial(self.ledger.get_streams, wallet=wallet, accounts=wallet.accounts) - stream_count = partial(self.ledger.get_stream_count, wallet=wallet, accounts=wallet.accounts) - return paginate_rows(streams, stream_count, page, page_size, resolve=resolve) + kwargs['type'] = 'stream' + kwargs['unspent'] = True + return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DHT_COMPONENT, DATABASE_COMPONENT) @@ -3912,15 +3902,18 @@ class Daemon(metaclass=JSONRPCServerType): return tx @requires(WALLET_COMPONENT) - def jsonrpc_support_list(self, account_id=None, wallet_id=None, page=None, page_size=None): + def jsonrpc_support_list(self, *args, **kwargs): """ List supports and tips in my control. Usage: support_list [ | --account_id=] [--wallet_id=] + [--name=...] [--claim_id=...] [--page=] [--page_size=] Options: + --name= : (str or list) claim name + --claim_id= : (str or list) claim id --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -3928,15 +3921,9 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {Paginated[Output]} """ - wallet = self.wallet_manager.get_wallet_or_default(wallet_id) - if account_id: - account = wallet.get_account_or_error(account_id) - supports = account.get_supports - support_count = account.get_support_count - else: - supports = partial(self.ledger.get_supports, wallet=wallet, accounts=wallet.accounts) - support_count = partial(self.ledger.get_support_count, wallet=wallet, accounts=wallet.accounts) - return paginate_rows(supports, support_count, page, page_size) + kwargs['type'] = 'support' + kwargs['unspent'] = True + return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) async def jsonrpc_support_abandon( @@ -4103,12 +4090,60 @@ class Daemon(metaclass=JSONRPCServerType): """ return self.wallet_manager.get_transaction(txid) + TXO_DOC = """ + List transaction outputs. + """ + + @requires(WALLET_COMPONENT) + def jsonrpc_txo_list( + self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin + claim_id=None, name=None, unspent=False, + wallet_id=None, page=None, page_size=None, resolve=False): + """ + List my transaction outputs. + + Usage: + txo_list [--account_id=] [--type=...] [--txid=...] + [--claim_id=...] [--name=...] [--unspent] + [--wallet_id=] + [--page=] [--page_size=] + [--resolve] + + Options: + --type= : (str or list) claim type: stream, channel, support, + purchase, collection, repost, other + --txid= : (str or list) transaction id of outputs + --unspent : (bool) hide spent outputs, show only unspent ones + --claim_id= : (str or list) claim id + --name= : (str or list) claim name + --account_id= : (str) id of the account to query + --wallet_id= : (str) restrict results to specific wallet + --page= : (int) page to return during paginating + --page_size= : (int) number of items on page during pagination + --resolve : (bool) resolves each claim to provide additional metadata + + Returns: {Paginated[Output]} + """ + wallet = self.wallet_manager.get_wallet_or_default(wallet_id) + if account_id: + account = wallet.get_account_or_error(account_id) + claims = account.get_txos + claim_count = account.get_txo_count + else: + claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts) + claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts) + constraints = {'resolve': resolve, 'unspent': unspent} + database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) + database.constrain_single_or_list(constraints, 'claim_id', claim_id) + database.constrain_single_or_list(constraints, 'claim_name', name) + return paginate_rows(claims, claim_count, page, page_size, **constraints) + UTXO_DOC = """ Unspent transaction management. """ @requires(WALLET_COMPONENT) - def jsonrpc_utxo_list(self, account_id=None, wallet_id=None, page=None, page_size=None): + def jsonrpc_utxo_list(self, *args, **kwargs): """ List unspent transaction outputs @@ -4124,15 +4159,9 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {Paginated[Output]} """ - wallet = self.wallet_manager.get_wallet_or_default(wallet_id) - if account_id: - account = wallet.get_account_or_error(account_id) - utxos = account.get_utxos - utxo_count = account.get_utxo_count - else: - utxos = partial(self.ledger.get_utxos, wallet=wallet, accounts=wallet.accounts) - utxo_count = partial(self.ledger.get_utxo_count, wallet=wallet, accounts=wallet.accounts) - return paginate_rows(utxos, utxo_count, page, page_size) + kwargs['type'] = ['other', 'purchase'] + kwargs['unspent'] = True + return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) async def jsonrpc_utxo_release(self, account_id=None, wallet_id=None): diff --git a/lbry/testcase.py b/lbry/testcase.py index fe86a7ee0..418adf2a2 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -564,6 +564,9 @@ class CommandTestCase(IntegrationTestCase): async def file_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items'] + async def txo_list(self, *args, **kwargs): + return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items'] + async def claim_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items'] @@ -573,6 +576,9 @@ class CommandTestCase(IntegrationTestCase): async def channel_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items'] + async def transaction_list(self, *args, **kwargs): + return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items'] + @staticmethod def get_claim_id(tx): return tx['outputs'][0]['claim_id'] diff --git a/lbry/wallet/account.py b/lbry/wallet/account.py index 128266ea5..76ab7bb5a 100644 --- a/lbry/wallet/account.py +++ b/lbry/wallet/account.py @@ -467,6 +467,12 @@ class Account: 'max_receiving_gap': receiving_gap, } + def get_txos(self, **constraints): + return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints) + + def get_txo_count(self, **constraints): + return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints) + def get_utxos(self, **constraints): return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints) diff --git a/lbry/wallet/constants.py b/lbry/wallet/constants.py index 5935a6717..34b8baa88 100644 --- a/lbry/wallet/constants.py +++ b/lbry/wallet/constants.py @@ -6,6 +6,7 @@ COIN = 100*CENT TIMEOUT = 30.0 TXO_TYPES = { + "other": 0, "stream": 1, "channel": 2, "support": 3, @@ -14,9 +15,13 @@ TXO_TYPES = { "repost": 6, } -CLAIM_TYPES = [ - TXO_TYPES['stream'], - TXO_TYPES['channel'], - TXO_TYPES['collection'], - TXO_TYPES['repost'], +CLAIM_TYPE_NAMES = [ + 'stream', + 'channel', + 'collection', + 'repost', +] + +CLAIM_TYPES = [ + TXO_TYPES[name] for name in CLAIM_TYPE_NAMES ] diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 2c6ec8bc3..4e75c0a69 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -107,7 +107,9 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''): if not key: sql.append(constraint) continue - if key.startswith('$'): + if key.startswith('$$'): + col, key = col[2:], key[1:] + elif key.startswith('$'): values[key] = constraint continue if key.endswith('__not'): @@ -221,6 +223,19 @@ def rows_to_dict(rows, fields): return [] +def constrain_single_or_list(constraints, column, value, convert=lambda x: x): + if value is not None: + if isinstance(value, list): + value = [convert(v) for v in value] + if len(value) == 1: + constraints[column] = value[0] + elif len(value) > 1: + constraints[f"{column}__in"] = value + else: + constraints[column] = convert(value) + return constraints + + class SQLiteMixin: SCHEMA_VERSION: Optional[str] = None @@ -350,6 +365,7 @@ class Database(SQLiteMixin): create index if not exists txo_txid_idx on txo (txid); create index if not exists txo_address_idx on txo (address); create index if not exists txo_claim_id_idx on txo (claim_id); + create index if not exists txo_claim_name_idx on txo (claim_name); create index if not exists txo_txo_type_idx on txo (txo_type); """ @@ -474,16 +490,15 @@ class Database(SQLiteMixin): async def select_transactions(self, cols, accounts=None, **constraints): if not {'txid', 'txid__in'}.intersection(constraints): assert accounts, "'accounts' argument required when no 'txid' constraint is present" - constraints.update({ - f'$account{i}': a.public_key.address for i, a in enumerate(accounts) + where, values = constraints_to_sql({ + '$$account_address.account__in': [a.public_key.address for a in accounts] }) - account_values = ', '.join([f':$account{i}' for i in range(len(accounts))]) - where = f" WHERE account_address.account IN ({account_values})" constraints['txid__in'] = f""" - SELECT txo.txid FROM txo JOIN account_address USING (address) {where} + SELECT txo.txid FROM txo JOIN account_address USING (address) WHERE {where} UNION - SELECT txi.txid FROM txi JOIN account_address USING (address) {where} + SELECT txi.txid FROM txi JOIN account_address USING (address) WHERE {where} """ + constraints.update(values) return await self.db.execute_fetchall( *query(f"SELECT {cols} FROM tx", **constraints) ) @@ -568,7 +583,14 @@ class Database(SQLiteMixin): sql += " JOIN account_address USING (address)" return await self.db.execute_fetchall(*query(sql, **constraints)) - async def get_txos(self, wallet=None, no_tx=False, **constraints): + @staticmethod + def constrain_unspent(constraints): + constraints['is_reserved'] = False + constraints['txoid__not_in'] = "SELECT txoid FROM txi" + + async def get_txos(self, wallet=None, no_tx=False, unspent=False, **constraints): + if unspent: + self.constrain_unspent(constraints) my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set() if 'order_by' not in constraints: constraints['order_by'] = [ @@ -637,33 +659,28 @@ class Database(SQLiteMixin): return txos - async def get_txo_count(self, **constraints): + async def get_txo_count(self, unspent=False, **constraints): constraints.pop('resolve', None) constraints.pop('wallet', None) constraints.pop('offset', None) constraints.pop('limit', None) constraints.pop('order_by', None) + if unspent: + self.constrain_unspent(constraints) count = await self.select_txos('count(*)', **constraints) return count[0][0] - @staticmethod - def constrain_utxo(constraints): - constraints['is_reserved'] = False - constraints['txoid__not_in'] = "SELECT txoid FROM txi" - def get_utxos(self, **constraints): - self.constrain_utxo(constraints) - return self.get_txos(**constraints) + return self.get_txos(unspent=True, **constraints) def get_utxo_count(self, **constraints): - self.constrain_utxo(constraints) - return self.get_txo_count(**constraints) + return self.get_txo_count(unspent=True, **constraints) async def get_balance(self, wallet=None, accounts=None, **constraints): assert wallet or accounts, \ "'wallet' or 'accounts' constraints required to calculate balance" constraints['accounts'] = accounts or wallet.accounts - self.constrain_utxo(constraints) + self.constrain_unspent(constraints) balance = await self.select_txos('SUM(amount)', **constraints) return balance[0][0] or 0 @@ -746,11 +763,13 @@ class Database(SQLiteMixin): @staticmethod def constrain_claims(constraints): + if {'txo_type', 'txo_type__in'}.intersection(constraints): + return claim_types = constraints.pop('claim_type', None) - if isinstance(claim_types, str) and claim_types: - claim_types = [claim_types] - if isinstance(claim_types, list) and claim_types: - constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types] + if claim_types: + constrain_single_or_list( + constraints, 'txo_type', claim_types, lambda x: TXO_TYPES[x] + ) else: constraints['txo_type__in'] = CLAIM_TYPES diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 6215c7bd2..feef913b5 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -262,6 +262,15 @@ class Ledger(metaclass=LedgerRegistry): self.constraint_spending_utxos(constraints) return self.db.get_utxo_count(**constraints) + async def get_txos(self, resolve=False, **constraints): + txos = await self.db.get_txos(**constraints) + if resolve: + return await self._resolve_for_local_results(constraints.get('accounts', []), txos) + return txos + + def get_txo_count(self, **constraints): + return self.db.get_txo_count(**constraints) + def get_transactions(self, **constraints): return self.db.get_transactions(**constraints) @@ -735,9 +744,11 @@ class Ledger(metaclass=LedgerRegistry): async def _resolve_for_local_results(self, accounts, txos): results = [] - response = await self.resolve(accounts, [txo.permanent_url for txo in txos]) + response = await self.resolve( + accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim] + ) for txo in txos: - resolved = response[txo.permanent_url] + resolved = response.get(txo.permanent_url) if txo.can_decode_claim else None if isinstance(resolved, Output): resolved.update_annotations(txo) results.append(resolved) diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index 8b5b76aee..7edd0b823 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -1,4 +1,3 @@ -import asyncio import os.path import tempfile import logging @@ -399,12 +398,96 @@ class ClaimSearchCommand(ClaimTestCase): await self.assertFindsClaims([], text='cloud') +class TransactionCommands(ClaimTestCase): + + async def test_transaction_list(self): + channel_id = self.get_claim_id(await self.channel_create()) + await self.channel_update(channel_id, bid='0.5') + await self.channel_abandon(claim_id=channel_id) + stream_id = self.get_claim_id(await self.stream_create()) + await self.stream_update(stream_id, bid='0.5') + await self.stream_abandon(claim_id=stream_id) + + r = await self.transaction_list() + self.assertEqual(7, len(r)) + self.assertEqual(stream_id, r[0]['abandon_info'][0]['claim_id']) + self.assertEqual(stream_id, r[1]['update_info'][0]['claim_id']) + self.assertEqual(stream_id, r[2]['claim_info'][0]['claim_id']) + self.assertEqual(channel_id, r[3]['abandon_info'][0]['claim_id']) + self.assertEqual(channel_id, r[4]['update_info'][0]['claim_id']) + self.assertEqual(channel_id, r[5]['claim_info'][0]['claim_id']) + + +class TransactionOutputCommands(ClaimTestCase): + + async def test_txo_list_filtering(self): + channel_id = self.get_claim_id(await self.channel_create()) + await self.channel_update(channel_id, bid='0.5') + stream_id = self.get_claim_id(await self.stream_create()) + await self.stream_update(stream_id, bid='0.5') + + # type filtering + r = await self.txo_list(type='channel') + self.assertEqual(2, len(r)) + self.assertEqual('channel', r[0]['value_type']) + self.assertFalse(r[0]['is_spent']) + self.assertEqual('channel', r[1]['value_type']) + self.assertTrue(r[1]['is_spent']) + + r = await self.txo_list(type='stream') + self.assertEqual(2, len(r)) + self.assertEqual('stream', r[0]['value_type']) + self.assertFalse(r[0]['is_spent']) + self.assertEqual('stream', r[1]['value_type']) + self.assertTrue(r[1]['is_spent']) + + r = await self.txo_list(type=['stream', 'channel']) + self.assertEqual(4, len(r)) + self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r}) + + # claim_id filtering + r = await self.txo_list(claim_id=stream_id) + self.assertEqual(2, len(r)) + self.assertEqual({stream_id}, {c['claim_id'] for c in r}) + + r = await self.txo_list(claim_id=[stream_id, channel_id]) + self.assertEqual(4, len(r)) + self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) + stream_name, _, channel_name, _ = (c['name'] for c in r) + + r = await self.txo_list(claim_id=['beef']) + self.assertEqual(0, len(r)) + + # claim_name filtering + r = await self.txo_list(name=stream_name) + self.assertEqual(2, len(r)) + self.assertEqual({stream_id}, {c['claim_id'] for c in r}) + + r = await self.txo_list(name=[stream_name, channel_name]) + self.assertEqual(4, len(r)) + self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) + + r = await self.txo_list(name=['beef']) + self.assertEqual(0, len(r)) + + r = await self.txo_list() + self.assertEqual(9, len(r)) + await self.stream_abandon(claim_id=stream_id) + r = await self.txo_list() + self.assertEqual(10, len(r)) + r = await self.txo_list(claim_id=stream_id) + self.assertEqual(2, len(r)) + self.assertTrue(r[0]['is_spent']) + self.assertTrue(r[1]['is_spent']) + + class ClaimCommands(ClaimTestCase): - async def test_claim_list_type_filtering(self): - await self.channel_create() - await self.stream_create() + async def test_claim_list_filtering(self): + channel_id = self.get_claim_id(await self.channel_create()) + stream_id = self.get_claim_id(await self.stream_create()) + # type filtering r = await self.claim_list(claim_type='channel') self.assertEqual(1, len(r)) self.assertEqual('channel', r[0]['value_type']) @@ -417,6 +500,31 @@ class ClaimCommands(ClaimTestCase): self.assertEqual(2, len(r)) self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r}) + # claim_id filtering + r = await self.claim_list(claim_id=stream_id) + self.assertEqual(1, len(r)) + self.assertEqual({stream_id}, {c['claim_id'] for c in r}) + + r = await self.claim_list(claim_id=[stream_id, channel_id]) + self.assertEqual(2, len(r)) + self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) + stream_name, channel_name = (c['name'] for c in r) + + r = await self.claim_list(claim_id=['beef']) + self.assertEqual(0, len(r)) + + # claim_name filtering + r = await self.claim_list(name=stream_name) + self.assertEqual(1, len(r)) + self.assertEqual({stream_id}, {c['claim_id'] for c in r}) + + r = await self.claim_list(name=[stream_name, channel_name]) + self.assertEqual(2, len(r)) + self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) + + r = await self.claim_list(name=['beef']) + self.assertEqual(0, len(r)) + async def test_claim_stream_channel_list_with_resolve(self): self.assertListEqual([], await self.claim_list(resolve=True)) @@ -1364,7 +1472,7 @@ class StreamCommands(ClaimTestCase): tx = await self.stream_create(bid='2.5') # creates new claim claim_id = self.get_claim_id(tx) - txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items'] + txs = await self.transaction_list() self.assertEqual(len(txs[0]['claim_info']), 1) self.assertEqual(txs[0]['confirmations'], 1) self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5') @@ -1379,7 +1487,7 @@ class StreamCommands(ClaimTestCase): self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) await self.stream_update(claim_id, bid='1.0') # updates previous claim - txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items'] + txs = await self.transaction_list() self.assertEqual(len(txs[0]['update_info']), 1) self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5') self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id) @@ -1390,7 +1498,7 @@ class StreamCommands(ClaimTestCase): await self.assertBalance(self.account, '8.9796765') await self.stream_abandon(claim_id) - txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items'] + txs = await self.transaction_list() self.assertEqual(len(txs[0]['abandon_info']), 1) self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0') self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id) @@ -1494,7 +1602,7 @@ class SupportCommands(CommandTestCase): await self.assertBalance(account2, '3.9998585') # verify that the incoming tip is marked correctly as is_tip=True in account1 - txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items'] + txs = await self.transaction_list(account_id=self.account.id) self.assertEqual(len(txs[0]['support_info']), 1) self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0') self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id) @@ -1504,9 +1612,7 @@ class SupportCommands(CommandTestCase): self.assertEqual(txs[0]['fee'], '0.0') # verify that the outgoing tip is marked correctly as is_tip=True in account2 - txs2 = (await self.out( - self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id) - ))['items'] + txs2 = await self.transaction_list(wallet_id='wallet2', account_id=account2.id) self.assertEqual(len(txs2[0]['support_info']), 1) self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0') self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id) @@ -1527,7 +1633,7 @@ class SupportCommands(CommandTestCase): await self.assertBalance(account2, '1.999717') # verify that the outgoing support is marked correctly as is_tip=False in account2 - txs2 = (await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')))['items'] + txs2 = await self.transaction_list(wallet_id='wallet2') self.assertEqual(len(txs2[0]['support_info']), 1) self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0') self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id) @@ -1539,7 +1645,7 @@ class SupportCommands(CommandTestCase): # abandoning the tip increases balance and shows tip as spent await self.support_abandon(claim_id) await self.assertBalance(self.account, '4.979662') - txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items'] + txs = await self.transaction_list(account_id=self.account.id) self.assertEqual(len(txs[0]['abandon_info']), 1) self.assertEqual(len(txs[1]['support_info']), 1) self.assertTrue(txs[1]['support_info'][0]['is_tip']) From 8ef2647fa95ec6c3b259804c1cd502d3279e8cde Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sun, 8 Mar 2020 23:11:03 -0400 Subject: [PATCH 2/2] is_received --- lbry/extras/daemon/daemon.py | 23 ++++++++++++++--- lbry/extras/daemon/json_response_encoder.py | 5 +++- lbry/wallet/database.py | 25 +++++++++++++++---- lbry/wallet/transaction.py | 5 ++-- .../blockchain/test_claim_commands.py | 20 +++++++++++++++ 5 files changed, 66 insertions(+), 12 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 1e796e4d7..c91c72f14 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -3902,18 +3902,19 @@ class Daemon(metaclass=JSONRPCServerType): return tx @requires(WALLET_COMPONENT) - def jsonrpc_support_list(self, *args, **kwargs): + def jsonrpc_support_list(self, *args, tips=None, **kwargs): """ List supports and tips in my control. Usage: support_list [ | --account_id=] [--wallet_id=] - [--name=...] [--claim_id=...] + [--name=...] [--claim_id=...] [--tips] [--page=] [--page_size=] Options: --name= : (str or list) claim name --claim_id= : (str or list) claim id + --tips : (bool) only show tips (is_received=true) --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -3923,6 +3924,9 @@ class Daemon(metaclass=JSONRPCServerType): """ kwargs['type'] = 'support' kwargs['unspent'] = True + kwargs['include_is_received'] = True + if tips is True: + kwargs['is_received'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) @@ -4098,6 +4102,7 @@ class Daemon(metaclass=JSONRPCServerType): def jsonrpc_txo_list( self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin claim_id=None, name=None, unspent=False, + include_is_received=False, is_received=None, is_not_received=None, wallet_id=None, page=None, page_size=None, resolve=False): """ List my transaction outputs. @@ -4105,7 +4110,8 @@ class Daemon(metaclass=JSONRPCServerType): Usage: txo_list [--account_id=] [--type=...] [--txid=...] [--claim_id=...] [--name=...] [--unspent] - [--wallet_id=] + [--include_is_received] [--is_received] [--is_not_received] + [--wallet_id=] [--include_is_received] [--is_received] [--page=] [--page_size=] [--resolve] @@ -4116,6 +4122,11 @@ class Daemon(metaclass=JSONRPCServerType): --unspent : (bool) hide spent outputs, show only unspent ones --claim_id= : (str or list) claim id --name= : (str or list) claim name + --include_is_received : (bool) calculate the is_received property and + include in output, this happens automatically if you + use the --is_received or --is_not_received filters + --is_received : (bool) only return txos sent from others to this account + --is_not_received : (bool) only return txos created by this account --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -4132,7 +4143,11 @@ class Daemon(metaclass=JSONRPCServerType): else: claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts) claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts) - constraints = {'resolve': resolve, 'unspent': unspent} + constraints = {'resolve': resolve, 'unspent': unspent, 'include_is_received': include_is_received} + if is_received is True: + constraints['is_received'] = True + elif is_not_received is True: + constraints['is_received'] = False database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) database.constrain_single_or_list(constraints, 'claim_id', claim_id) database.constrain_single_or_list(constraints, 'claim_name', name) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index 4c400f8f6..a3c9e318d 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -25,7 +25,8 @@ def encode_txo_doc(): 'address': "address of who can spend the txo", 'confirmations': "number of confirmed blocks", 'is_change': "payment to change address, only available when it can be determined", - 'is_spent': "true if txo is spent, false or None if it could not be determined", + 'is_received': "true if txo was sent from external account to this account", + 'is_spent': "true if txo is spent", 'is_mine': "payment to one of your accounts, only available when it can be determined", 'type': "one of 'claim', 'support' or 'purchase'", 'name': "when type is 'claim' or 'support', this is the claim name", @@ -169,6 +170,8 @@ class JSONResponseEncoder(JSONEncoder): } if txo.is_change is not None: output['is_change'] = txo.is_change + if txo.is_received is not None: + output['is_received'] = txo.is_received if txo.is_spent is not None: output['is_spent'] = txo.is_spent if txo.is_my_account is not None: diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 4e75c0a69..95e0e82c2 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -577,7 +577,19 @@ class Database(SQLiteMixin): if txs: return txs[0] - async def select_txos(self, cols, **constraints): + async def select_txos(self, cols, wallet=None, include_is_received=False, **constraints): + if include_is_received: + assert wallet is not None, 'cannot use is_recieved filter without wallet argument' + account_in_wallet, values = constraints_to_sql({ + '$$account__in#is_received': [a.public_key.address for a in wallet.accounts] + }) + cols += f""", + NOT EXISTS( + SELECT 1 FROM txi JOIN account_address USING (address) + WHERE txi.txid=txo.txid AND {account_in_wallet} + ) as is_received + """ + constraints.update(values) sql = f"SELECT {cols} FROM txo JOIN tx USING (txid)" if 'accounts' in constraints: sql += " JOIN account_address USING (address)" @@ -588,7 +600,8 @@ class Database(SQLiteMixin): constraints['is_reserved'] = False constraints['txoid__not_in'] = "SELECT txoid FROM txi" - async def get_txos(self, wallet=None, no_tx=False, unspent=False, **constraints): + async def get_txos(self, wallet=None, no_tx=False, unspent=False, include_is_received=False, **constraints): + include_is_received = include_is_received or 'is_received' in constraints if unspent: self.constrain_unspent(constraints) my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set() @@ -601,9 +614,9 @@ class Database(SQLiteMixin): tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, ( select group_concat(account||"|"||chain) from account_address where account_address.address=txo.address - ), exists(select txoid from txi where txi.txoid=txo.txoid) + ), exists(select 1 from txi where txi.txoid=txo.txoid) """, - **constraints + wallet=wallet, include_is_received=include_is_received, **constraints ) txos = [] txs = {} @@ -624,6 +637,8 @@ class Database(SQLiteMixin): row_accounts = dict(a.split('|') for a in row[8].split(',')) account_match = set(row_accounts) & my_accounts txo.is_spent = bool(row[9]) + if include_is_received: + txo.is_received = bool(row[10]) if account_match: txo.is_my_account = True txo.is_change = row_accounts[account_match.pop()] == '1' @@ -660,8 +675,8 @@ class Database(SQLiteMixin): return txos async def get_txo_count(self, unspent=False, **constraints): + constraints['include_is_received'] = 'is_received' in constraints constraints.pop('resolve', None) - constraints.pop('wallet', None) constraints.pop('offset', None) constraints.pop('limit', None) constraints.pop('order_by', None) diff --git a/lbry/wallet/transaction.py b/lbry/wallet/transaction.py index 882e93df4..cfd637687 100644 --- a/lbry/wallet/transaction.py +++ b/lbry/wallet/transaction.py @@ -207,7 +207,7 @@ class OutputEffectiveAmountEstimator: class Output(InputOutput): __slots__ = ( - 'amount', 'script', 'is_change', 'is_spent', 'is_my_account', + 'amount', 'script', 'is_change', 'is_spent', 'is_received', 'is_my_account', 'channel', 'private_key', 'meta', 'purchase', 'purchased_claim', 'purchase_receipt', 'reposted_claim', 'claims', @@ -216,7 +216,7 @@ class Output(InputOutput): def __init__(self, amount: int, script: OutputScript, tx_ref: TXRef = None, position: int = None, is_change: Optional[bool] = None, is_spent: Optional[bool] = None, - is_my_account: Optional[bool] = None, + is_received: Optional[bool] = None, is_my_account: Optional[bool] = None, channel: Optional['Output'] = None, private_key: Optional[str] = None ) -> None: super().__init__(tx_ref, position) @@ -224,6 +224,7 @@ class Output(InputOutput): self.script = script self.is_change = is_change self.is_spent = is_spent + self.is_received = is_received self.is_my_account = is_my_account self.channel = channel self.private_key = private_key diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index 7edd0b823..a99106751 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -480,6 +480,26 @@ class TransactionOutputCommands(ClaimTestCase): self.assertTrue(r[0]['is_spent']) self.assertTrue(r[1]['is_spent']) + async def test_txo_list_received_filtering(self): + wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True) + address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id) + await self.channel_create(claim_address=address2) + + r = await self.txo_list(include_is_received=True) + self.assertEqual(2, len(r)) + self.assertFalse(r[0]['is_received']) + self.assertTrue(r[1]['is_received']) + rt = await self.txo_list(is_not_received=True) + self.assertEqual(1, len(rt)) + self.assertEqual(rt[0], r[0]) + rf = await self.txo_list(is_received=True) + self.assertEqual(1, len(rf)) + self.assertEqual(rf[0], r[1]) + + r = await self.txo_list(include_is_received=True, wallet_id=wallet2.id) + self.assertEqual(1, len(r)) + self.assertTrue(r[0]['is_received']) + class ClaimCommands(ClaimTestCase):