From af2f2282c27432f94e02182906a3e76353499a28 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Wed, 18 Mar 2020 00:15:24 -0400 Subject: [PATCH 01/11] txo_list returns txo funded by my account but sent to external address --- lbry/extras/daemon/daemon.py | 22 +++++-- lbry/testcase.py | 5 ++ lbry/wallet/database.py | 113 +++++++++++++++++++++++------------ 3 files changed, 96 insertions(+), 44 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 51507a526..0f536e747 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4114,7 +4114,8 @@ class Daemon(metaclass=JSONRPCServerType): def jsonrpc_txo_list( self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin claim_id=None, name=None, unspent=False, - include_is_received=False, is_received=None, is_not_received=None, + include_is_my_output=False, is_my_output=None, is_not_my_output=None, + include_is_my_input=False, is_my_input=None, is_not_my_input=None, wallet_id=None, page=None, page_size=None, resolve=False): """ List my transaction outputs. @@ -4155,11 +4156,20 @@ class Daemon(metaclass=JSONRPCServerType): else: claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts, read_only=True) claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts, read_only=True) - constraints = {'resolve': resolve, 'unspent': unspent, 'include_is_received': include_is_received} - if is_received is True: - constraints['is_received'] = True - elif is_not_received is True: - constraints['is_received'] = False + constraints = { + 'resolve': resolve, + 'unspent': unspent, + 'include_is_my_input': include_is_my_input, + 'include_is_my_output': include_is_my_output + } + if is_my_input is True: + constraints['is_my_input'] = True + elif is_not_my_input is True: + constraints['is_my_input'] = False + if is_my_output is True: + constraints['is_my_output'] = True + elif is_not_my_output is True: + constraints['is_my_output'] = False database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) database.constrain_single_or_list(constraints, 'claim_id', claim_id) database.constrain_single_or_list(constraints, 'claim_name', name) diff --git a/lbry/testcase.py b/lbry/testcase.py index 418adf2a2..483511ae9 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -555,6 +555,11 @@ class CommandTestCase(IntegrationTestCase): self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm ) + async def wallet_send(self, *args, confirm=True, **kwargs): + return await self.confirm_and_render( + self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm + ) + async def resolve(self, uri): return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri] diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 4c733fa14..46ed6f57a 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -450,11 +450,12 @@ class Database(SQLiteMixin): CREATE_TXI_TABLE = """ create table if not exists txi ( txid text references tx, - txoid text references txo, - address text references pubkey_address + txoid text references txo primary key, + address text references pubkey_address, + position integer not null ); create index if not exists txi_address_idx on txi (address); - create index if not exists txi_txoid_idx on txi (txoid); + create index if not exists first_input_idx on txi (txid, address) where position=0; """ CREATE_TABLES_QUERY = ( @@ -466,12 +467,11 @@ class Database(SQLiteMixin): CREATE_TXI_TABLE ) - @staticmethod - def txo_to_row(tx, address, txo): + def txo_to_row(self, tx, txo): row = { 'txid': tx.id, 'txoid': txo.id, - 'address': address, + 'address': txo.get_address(self.ledger), 'position': txo.position, 'amount': txo.amount, 'script': sqlite3.Binary(txo.script.source) @@ -517,25 +517,29 @@ class Database(SQLiteMixin): def _transaction_io(self, conn: sqlite3.Connection, tx: Transaction, address, txhash): conn.execute(*self._insert_sql('tx', self.tx_to_row(tx), replace=True)).fetchall() - for txo in tx.outputs: - if txo.script.is_pay_pubkey_hash and txo.pubkey_hash == txhash: - conn.execute(*self._insert_sql( - "txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True - )).fetchall() - elif txo.script.is_pay_script_hash: - # TODO: implement script hash payments - log.warning('Database.save_transaction_io: pay script hash is not implemented!') + is_my_input = False for txi in tx.inputs: if txi.txo_ref.txo is not None: txo = txi.txo_ref.txo if txo.has_address and txo.get_address(self.ledger) == address: + is_my_input = True conn.execute(*self._insert_sql("txi", { 'txid': tx.id, 'txoid': txo.id, 'address': address, + 'position': txi.position }, ignore_duplicate=True)).fetchall() + for txo in tx.outputs: + if txo.script.is_pay_pubkey_hash and (txo.pubkey_hash == txhash or is_my_input): + conn.execute(*self._insert_sql( + "txo", self.txo_to_row(tx, txo), ignore_duplicate=True + )).fetchall() + elif txo.script.is_pay_script_hash: + # TODO: implement script hash payments + log.warning('Database.save_transaction_io: pay script hash is not implemented!') + def save_transaction_io(self, tx: Transaction, address, txhash, history): return self.save_transaction_io_batch([tx], address, txhash, history) @@ -655,22 +659,44 @@ class Database(SQLiteMixin): if txs: return txs[0] - async def select_txos(self, cols, wallet=None, include_is_received=False, read_only=False, **constraints): - if include_is_received: - assert wallet is not None, 'cannot use is_recieved filter without wallet argument' - account_in_wallet, values = constraints_to_sql({ - '$$account__in#is_received': [a.public_key.address for a in wallet.accounts] + async def select_txos(self, cols, accounts=None, is_my_input=None, is_my_output=True, read_only=True, **constraints): + if 'txoid' in constraints: + constraints['txo.txoid'] = constraints.pop('txoid') + if 'txoid__in' in constraints: + constraints['txo.txoid__in'] = constraints.pop('txoid__in') + if accounts: + account_in_sql, values = constraints_to_sql({ + '$$account__in': [a.public_key.address for a in accounts] }) - cols += f""", - NOT EXISTS( - SELECT 1 FROM txi JOIN account_address USING (address) - WHERE txi.txid=txo.txid AND {account_in_wallet} - ) as is_received - """ + my_addresses = f"SELECT address FROM account_address WHERE {account_in_sql}" constraints.update(values) - sql = f"SELECT {cols} FROM txo JOIN tx USING (txid)" - if 'accounts' in constraints: - sql += " JOIN account_address USING (address)" + if is_my_input is True and is_my_output is True: # special case + constraints['received_or_sent__or'] = { + 'txo.address__in': my_addresses, + 'sent__and': { + 'txi.address__is_not_null': True, + 'txi.address__in': my_addresses + } + } + else: + if is_my_output is True: + constraints['txo.address__in'] = my_addresses + elif is_my_output is False: + constraints['txo.address__not_in'] = my_addresses + if is_my_input is True: + constraints['txi.address__is_not_null'] = True + constraints['txi.address__in'] = my_addresses + elif is_my_input is False: + constraints['is_my_input_false__or'] = { + 'txi.address__is_null': True, + 'txi.address__not_in': my_addresses + } + sql = f""" + SELECT {cols} FROM txo + JOIN tx ON (tx.txid=txo.txid) + LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid) + LEFT JOIN txi AS spent ON (spent.txoid=txo.txoid) + """ return await self.db.execute_fetchall(*query(sql, **constraints), read_only=read_only) @staticmethod @@ -678,24 +704,33 @@ class Database(SQLiteMixin): constraints['is_reserved'] = False constraints['txoid__not_in'] = "SELECT txoid FROM txi" - async def get_txos(self, wallet=None, no_tx=False, unspent=False, include_is_received=False, - read_only=False, **constraints): - include_is_received = include_is_received or 'is_received' in constraints + async def get_txos( + self, wallet=None, no_tx=False, unspent=False, + include_is_my_input=False, include_is_my_output=False, + read_only=False, **constraints): if unspent: self.constrain_unspent(constraints) my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set() + is_my_input_column = "" + if include_is_my_input and my_accounts: + account_in_wallet_sql, values = constraints_to_sql({'$$account__in#_wallet': my_accounts}) + is_my_input_column = f""", ( + txi.address IS NULL AND + txi.address IN (SELECT address FROM account_address WHERE {account_in_wallet_sql}) + ) + """ + constraints.update(values) if 'order_by' not in constraints: constraints['order_by'] = [ "tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position" ] rows = await self.select_txos( - """ + f""" tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, ( select group_concat(account||"|"||chain) from account_address where account_address.address=txo.address - ), exists(select 1 from txi where txi.txoid=txo.txoid) - """, - wallet=wallet, include_is_received=include_is_received, read_only=read_only, **constraints + ), spent.txoid IS NOT NULL {is_my_input_column} + """, read_only=read_only, **constraints ) txos = [] txs = {} @@ -716,8 +751,8 @@ class Database(SQLiteMixin): row_accounts = dict(a.split('|') for a in row[8].split(',')) account_match = set(row_accounts) & my_accounts txo.is_spent = bool(row[9]) - if include_is_received: - txo.is_received = bool(row[10]) + if include_is_my_input and my_accounts: + txo.is_received = not bool(row[10]) if account_match: txo.is_my_account = True txo.is_change = row_accounts[account_match.pop()] == '1' @@ -755,7 +790,9 @@ class Database(SQLiteMixin): return txos async def get_txo_count(self, unspent=False, **constraints): - constraints['include_is_received'] = 'is_received' in constraints + constraints.pop('include_is_my_input', None) + constraints.pop('include_is_my_output', None) + constraints.pop('wallet', None) constraints.pop('resolve', None) constraints.pop('offset', None) constraints.pop('limit', None) From dd2180359807ab894061fd24960cac6d8b245c6c Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 20 Mar 2020 01:11:05 -0400 Subject: [PATCH 02/11] working --is_my_input_or_output --is_my_input --is_my_output etc --- lbry/extras/daemon/daemon.py | 61 +++--- lbry/extras/daemon/json_response_encoder.py | 12 +- lbry/wallet/account.py | 2 +- lbry/wallet/database.py | 202 +++++++++++------- lbry/wallet/ledger.py | 7 +- lbry/wallet/transaction.py | 60 +++--- .../blockchain/test_claim_commands.py | 76 +++++-- tests/unit/wallet/test_database.py | 28 +-- tests/unit/wallet/test_transaction.py | 24 +-- 9 files changed, 292 insertions(+), 180 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 0f536e747..59f1b0750 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -3926,7 +3926,7 @@ class Daemon(metaclass=JSONRPCServerType): Options: --name= : (str or list) claim name --claim_id= : (str or list) claim id - --tips : (bool) only show tips (is_received=true) + --tips : (bool) only show tips --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -3936,9 +3936,8 @@ class Daemon(metaclass=JSONRPCServerType): """ kwargs['type'] = 'support' kwargs['unspent'] = True - kwargs['include_is_received'] = True if tips is True: - kwargs['is_received'] = True + kwargs['is_my_output'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) @@ -4114,8 +4113,9 @@ class Daemon(metaclass=JSONRPCServerType): def jsonrpc_txo_list( self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin claim_id=None, name=None, unspent=False, - include_is_my_output=False, is_my_output=None, is_not_my_output=None, - include_is_my_input=False, is_my_input=None, is_not_my_input=None, + is_my_input_or_output=None, exclude_internal_transfers=False, + is_my_output=None, is_not_my_output=None, + is_my_input=None, is_not_my_input=None, wallet_id=None, page=None, page_size=None, resolve=False): """ List my transaction outputs. @@ -4123,23 +4123,31 @@ class Daemon(metaclass=JSONRPCServerType): Usage: txo_list [--account_id=] [--type=...] [--txid=...] [--claim_id=...] [--name=...] [--unspent] - [--include_is_received] [--is_received] [--is_not_received] - [--wallet_id=] [--include_is_received] [--is_received] - [--page=] [--page_size=] + [--is_my_input_or_output | + [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] + ] + [--exclude_internal_transfers] + [--wallet_id=] [--page=] [--page_size=] [--resolve] Options: --type= : (str or list) claim type: stream, channel, support, purchase, collection, repost, other --txid= : (str or list) transaction id of outputs - --unspent : (bool) hide spent outputs, show only unspent ones --claim_id= : (str or list) claim id --name= : (str or list) claim name - --include_is_received : (bool) calculate the is_received property and - include in output, this happens automatically if you - use the --is_received or --is_not_received filters - --is_received : (bool) only return txos sent from others to this account - --is_not_received : (bool) only return txos created by this account + --unspent : (bool) hide spent outputs, show only unspent ones + --is_my_input_or_output : (bool) txos which have your inputs or your outputs, + if using this flag the other related flags + are ignored (--is_my_output, --is_my_input, etc) + --is_my_output : (bool) show outputs controlled by you + --is_not_my_output : (bool) show outputs not controlled by you + --is_my_input : (bool) show outputs created by you + --is_not_my_input : (bool) show outputs not created by you + --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: + --is_my_input --is_my_output --type=other + this allows to exclude "change" payments, this + flag can be used in combination with any of the other flags --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating @@ -4159,17 +4167,22 @@ class Daemon(metaclass=JSONRPCServerType): constraints = { 'resolve': resolve, 'unspent': unspent, - 'include_is_my_input': include_is_my_input, - 'include_is_my_output': include_is_my_output + 'exclude_internal_transfers': exclude_internal_transfers, + 'include_is_spent': True, + 'include_is_my_input': True, + 'include_is_my_output': True, } - if is_my_input is True: - constraints['is_my_input'] = True - elif is_not_my_input is True: - constraints['is_my_input'] = False - if is_my_output is True: - constraints['is_my_output'] = True - elif is_not_my_output is True: - constraints['is_my_output'] = False + if is_my_input_or_output is True: + constraints['is_my_input_or_output'] = True + else: + if is_my_input is True: + constraints['is_my_input'] = True + elif is_not_my_input is True: + constraints['is_my_input'] = False + if is_my_output is True: + constraints['is_my_output'] = True + elif is_not_my_output is True: + constraints['is_my_output'] = False database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) database.constrain_single_or_list(constraints, 'claim_id', claim_id) database.constrain_single_or_list(constraints, 'claim_name', name) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index a3c9e318d..da87da92e 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -168,14 +168,14 @@ class JSONResponseEncoder(JSONEncoder): 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, 'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None } - if txo.is_change is not None: - output['is_change'] = txo.is_change - if txo.is_received is not None: - output['is_received'] = txo.is_received if txo.is_spent is not None: output['is_spent'] = txo.is_spent - if txo.is_my_account is not None: - output['is_mine'] = txo.is_my_account + if txo.is_my_output is not None: + output['is_my_output'] = txo.is_my_output + if txo.is_my_input is not None: + output['is_my_input'] = txo.is_my_input + if txo.is_internal_transfer is not None: + output['is_internal_transfer'] = txo.is_internal_transfer if txo.script.is_claim_name: output['type'] = 'claim' diff --git a/lbry/wallet/account.py b/lbry/wallet/account.py index b7d4e29c5..5194210d5 100644 --- a/lbry/wallet/account.py +++ b/lbry/wallet/account.py @@ -437,7 +437,7 @@ class Account: async def get_addresses(self, read_only=False, **constraints) -> List[str]: rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints) - return [r[0] for r in rows] + return [r['address'] for r in rows] def get_address_records(self, **constraints): return self.ledger.db.get_addresses(accounts=[self], **constraints) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 46ed6f57a..429c5d1c3 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -29,6 +29,7 @@ reader_context: Optional[ContextVar[ReaderProcessState]] = ContextVar('reader_co def initializer(path): db = sqlite3.connect(path) + db.row_factory = dict_row_factory db.executescript("pragma journal_mode=WAL;") reader = ReaderProcessState(db.cursor()) reader_context.set(reader) @@ -106,7 +107,7 @@ class AIOSQLite: return self.run(lambda conn: conn.executescript(script)) async def _execute_fetch(self, sql: str, parameters: Iterable = None, - read_only=False, fetch_all: bool = False) -> Iterable[sqlite3.Row]: + read_only=False, fetch_all: bool = False) -> List[dict]: read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone parameters = parameters if parameters is not None else [] if read_only: @@ -120,11 +121,11 @@ class AIOSQLite: return await self.run(lambda conn: conn.execute(sql, parameters).fetchone()) async def execute_fetchall(self, sql: str, parameters: Iterable = None, - read_only=False) -> Iterable[sqlite3.Row]: + read_only=False) -> List[dict]: return await self._execute_fetch(sql, parameters, read_only, fetch_all=True) async def execute_fetchone(self, sql: str, parameters: Iterable = None, - read_only=False) -> Iterable[sqlite3.Row]: + read_only=False) -> List[dict]: return await self._execute_fetch(sql, parameters, read_only, fetch_all=False) def execute(self, sql: str, parameters: Iterable = None) -> Awaitable[sqlite3.Cursor]: @@ -294,13 +295,6 @@ def interpolate(sql, values): return sql -def rows_to_dict(rows, fields): - if rows: - return [dict(zip(fields, r)) for r in rows] - else: - return [] - - def constrain_single_or_list(constraints, column, value, convert=lambda x: x): if value is not None: if isinstance(value, list): @@ -384,6 +378,13 @@ class SQLiteMixin: return sql, values +def dict_row_factory(cursor, row): + d = {} + for idx, col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + + class Database(SQLiteMixin): SCHEMA_VERSION = "1.1" @@ -467,6 +468,10 @@ class Database(SQLiteMixin): CREATE_TXI_TABLE ) + async def open(self): + await super().open() + self.db.writer_connection.row_factory = dict_row_factory + def txo_to_row(self, tx, txo): row = { 'txid': tx.id, @@ -585,9 +590,13 @@ class Database(SQLiteMixin): *query(f"SELECT {cols} FROM tx", **constraints), read_only=read_only ) - TXO_NOT_MINE = Output(None, None, is_my_account=False) + TXO_NOT_MINE = Output(None, None, is_my_output=False) async def get_transactions(self, wallet=None, **constraints): + include_is_spent = constraints.pop('include_is_spent', False) + include_is_my_input = constraints.pop('include_is_my_input', False) + include_is_my_output = constraints.pop('include_is_my_output', False) + tx_rows = await self.select_transactions( 'txid, raw, height, position, is_verified', order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]), @@ -599,9 +608,10 @@ class Database(SQLiteMixin): txids, txs, txi_txoids = [], [], [] for row in tx_rows: - txids.append(row[0]) + txids.append(row['txid']) txs.append(Transaction( - raw=row[1], height=row[2], position=row[3], is_verified=bool(row[4]) + raw=row['raw'], height=row['height'], position=row['position'], + is_verified=bool(row['is_verified']) )) for txi in txs[-1].inputs: txi_txoids.append(txi.txo_ref.id) @@ -614,6 +624,9 @@ class Database(SQLiteMixin): (await self.get_txos( wallet=wallet, txid__in=txids[offset:offset+step], + include_is_spent=include_is_spent, + include_is_my_input=include_is_my_input, + include_is_my_output=include_is_my_output, )) }) @@ -624,6 +637,7 @@ class Database(SQLiteMixin): (await self.get_txos( wallet=wallet, txoid__in=txi_txoids[offset:offset+step], + include_is_my_output=include_is_my_output, )) }) @@ -651,26 +665,31 @@ class Database(SQLiteMixin): constraints.pop('offset', None) constraints.pop('limit', None) constraints.pop('order_by', None) - count = await self.select_transactions('count(*)', **constraints) - return count[0][0] + count = await self.select_transactions('COUNT(*) as total', **constraints) + return count[0]['total'] async def get_transaction(self, **constraints): txs = await self.get_transactions(limit=1, **constraints) if txs: return txs[0] - async def select_txos(self, cols, accounts=None, is_my_input=None, is_my_output=True, read_only=True, **constraints): - if 'txoid' in constraints: - constraints['txo.txoid'] = constraints.pop('txoid') - if 'txoid__in' in constraints: - constraints['txo.txoid__in'] = constraints.pop('txoid__in') + async def select_txos( + self, cols, accounts=None, is_my_input=None, is_my_output=True, + is_my_input_or_output=None, exclude_internal_transfers=False, + include_is_spent=False, include_is_my_input=False, + read_only=True, **constraints): + for rename_col in ('txid', 'txoid'): + for rename_constraint in (rename_col, rename_col+'__in', rename_col+'__not_in'): + if rename_constraint in constraints: + constraints['txo.'+rename_constraint] = constraints.pop(rename_constraint) if accounts: account_in_sql, values = constraints_to_sql({ '$$account__in': [a.public_key.address for a in accounts] }) my_addresses = f"SELECT address FROM account_address WHERE {account_in_sql}" constraints.update(values) - if is_my_input is True and is_my_output is True: # special case + if is_my_input_or_output: + include_is_my_input = True constraints['received_or_sent__or'] = { 'txo.address__in': my_addresses, 'sent__and': { @@ -679,85 +698,116 @@ class Database(SQLiteMixin): } } else: - if is_my_output is True: + if is_my_output: constraints['txo.address__in'] = my_addresses elif is_my_output is False: constraints['txo.address__not_in'] = my_addresses - if is_my_input is True: + if is_my_input: + include_is_my_input = True constraints['txi.address__is_not_null'] = True constraints['txi.address__in'] = my_addresses elif is_my_input is False: + include_is_my_input = True constraints['is_my_input_false__or'] = { 'txi.address__is_null': True, 'txi.address__not_in': my_addresses } - sql = f""" - SELECT {cols} FROM txo - JOIN tx ON (tx.txid=txo.txid) - LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid) - LEFT JOIN txi AS spent ON (spent.txoid=txo.txoid) - """ - return await self.db.execute_fetchall(*query(sql, **constraints), read_only=read_only) + if exclude_internal_transfers: + include_is_my_input = True + constraints['exclude_internal_payments__or'] = { + 'txo.txo_type__not': TXO_TYPES['other'], + 'txi.address__is_null': True, + 'txi.address__not_in': my_addresses + } + sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"] + if include_is_spent: + sql.append("LEFT JOIN txi AS spent ON (spent.txoid=txo.txoid)") + if include_is_my_input: + sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)") + return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only) @staticmethod def constrain_unspent(constraints): constraints['is_reserved'] = False - constraints['txoid__not_in'] = "SELECT txoid FROM txi" + constraints['include_is_spent'] = True + constraints['spent.txoid__is_null'] = True + + async def get_txos(self, wallet=None, no_tx=False, unspent=False, read_only=False, **constraints): - async def get_txos( - self, wallet=None, no_tx=False, unspent=False, - include_is_my_input=False, include_is_my_output=False, - read_only=False, **constraints): if unspent: self.constrain_unspent(constraints) + + include_is_spent = constraints.get('include_is_spent', False) + include_is_my_input = constraints.get('include_is_my_input', False) + include_is_my_output = constraints.pop('include_is_my_output', False) + + select_columns = [ + "tx.txid, raw, tx.height, tx.position as tx_position, tx.is_verified, " + "txo_type, txo.position as txo_position, amount, script" + ] + my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set() - is_my_input_column = "" - if include_is_my_input and my_accounts: - account_in_wallet_sql, values = constraints_to_sql({'$$account__in#_wallet': my_accounts}) - is_my_input_column = f""", ( - txi.address IS NULL AND - txi.address IN (SELECT address FROM account_address WHERE {account_in_wallet_sql}) - ) - """ + my_accounts_sql = "" + if include_is_my_output or include_is_my_input: + my_accounts_sql, values = constraints_to_sql({'$$account__in#_wallet': my_accounts}) constraints.update(values) + + if include_is_my_output and my_accounts: + if constraints.get('is_my_output', None) in (True, False): + select_columns.append(f"{1 if constraints['is_my_output'] else 0} AS is_my_output") + else: + select_columns.append(f"""( + txo.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) + ) AS is_my_output""") + + if include_is_my_input and my_accounts: + if constraints.get('is_my_input', None) in (True, False): + select_columns.append(f"{1 if constraints['is_my_input'] else 0} AS is_my_input") + else: + select_columns.append(f"""( + txi.address IS NOT NULL AND + txi.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) + ) AS is_my_input + """) + + if include_is_spent: + select_columns.append("spent.txoid IS NOT NULL AS is_spent") + if 'order_by' not in constraints: constraints['order_by'] = [ "tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position" ] - rows = await self.select_txos( - f""" - tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, ( - select group_concat(account||"|"||chain) from account_address - where account_address.address=txo.address - ), spent.txoid IS NOT NULL {is_my_input_column} - """, read_only=read_only, **constraints - ) + + rows = await self.select_txos(', '.join(select_columns), read_only=read_only, **constraints) + txos = [] txs = {} for row in rows: if no_tx: txo = Output( - amount=row[6], - script=OutputScript(row[7]), - tx_ref=TXRefImmutable.from_id(row[0], row[2]), - position=row[5] + amount=row['amount'], + script=OutputScript(row['script']), + tx_ref=TXRefImmutable.from_id(row['txid'], row['height']), + position=row['txo_position'] ) else: - if row[0] not in txs: - txs[row[0]] = Transaction( - row[1], height=row[2], position=row[3], is_verified=row[4] + if row['txid'] not in txs: + txs[row['txid']] = Transaction( + row['raw'], height=row['height'], position=row['tx_position'], + is_verified=bool(row['is_verified']) ) - txo = txs[row[0]].outputs[row[5]] - row_accounts = dict(a.split('|') for a in row[8].split(',')) - account_match = set(row_accounts) & my_accounts - txo.is_spent = bool(row[9]) - if include_is_my_input and my_accounts: - txo.is_received = not bool(row[10]) - if account_match: - txo.is_my_account = True - txo.is_change = row_accounts[account_match.pop()] == '1' - else: - txo.is_change = txo.is_my_account = False + txo = txs[row['txid']].outputs[row['txo_position']] + if include_is_spent: + txo.is_spent = bool(row['is_spent']) + if include_is_my_input: + txo.is_my_input = bool(row['is_my_input']) + if include_is_my_output: + txo.is_my_output = bool(row['is_my_output']) + if include_is_my_input and include_is_my_output: + if txo.is_my_input and txo.is_my_output and row['txo_type'] == TXO_TYPES['other']: + txo.is_internal_transfer = True + else: + txo.is_internal_transfer = False txos.append(txo) channel_ids = set() @@ -799,8 +849,8 @@ class Database(SQLiteMixin): constraints.pop('order_by', None) if unspent: self.constrain_unspent(constraints) - count = await self.select_txos('count(*)', **constraints) - return count[0][0] + count = await self.select_txos('COUNT(*) as total', **constraints) + return count[0]['total'] def get_utxos(self, read_only=False, **constraints): return self.get_txos(unspent=True, read_only=read_only, **constraints) @@ -813,8 +863,8 @@ class Database(SQLiteMixin): "'wallet' or 'accounts' constraints required to calculate balance" constraints['accounts'] = accounts or wallet.accounts self.constrain_unspent(constraints) - balance = await self.select_txos('SUM(amount)', read_only=read_only, **constraints) - return balance[0][0] or 0 + balance = await self.select_txos('SUM(amount) as total', read_only=read_only, **constraints) + return balance[0]['total'] or 0 async def select_addresses(self, cols, read_only=False, **constraints): return await self.db.execute_fetchall(*query( @@ -827,7 +877,7 @@ class Database(SQLiteMixin): 'address', 'account', 'chain', 'history', 'used_times', 'pubkey', 'chain_code', 'n', 'depth' ) - addresses = rows_to_dict(await self.select_addresses(', '.join(cols), read_only=read_only, **constraints), cols) + addresses = await self.select_addresses(', '.join(cols), read_only=read_only, **constraints) if 'pubkey' in cols: for address in addresses: address['pubkey'] = PubKey( @@ -837,8 +887,8 @@ class Database(SQLiteMixin): return addresses async def get_address_count(self, cols=None, read_only=False, **constraints): - count = await self.select_addresses('count(*)', read_only=read_only, **constraints) - return count[0][0] + count = await self.select_addresses('COUNT(*) as total', read_only=read_only, **constraints) + return count[0]['total'] async def get_address(self, read_only=False, **constraints): addresses = await self.get_addresses(read_only=read_only, limit=1, **constraints) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 64ac744c0..5c1f8d902 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -826,7 +826,10 @@ class Ledger(metaclass=LedgerRegistry): return self.db.get_support_count(**constraints) async def get_transaction_history(self, read_only=False, **constraints): - txs: List[Transaction] = await self.db.get_transactions(read_only=read_only, **constraints) + txs: List[Transaction] = await self.db.get_transactions( + include_is_my_output=True, include_is_spent=True, + read_only=read_only, **constraints + ) headers = self.headers history = [] for tx in txs: # pylint: disable=too-many-nested-blocks @@ -842,7 +845,7 @@ class Ledger(metaclass=LedgerRegistry): 'abandon_info': [], 'purchase_info': [] } - is_my_inputs = all([txi.is_my_account for txi in tx.inputs]) + is_my_inputs = all([txi.is_my_input for txi in tx.inputs]) if is_my_inputs: # fees only matter if we are the ones paying them item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee) diff --git a/lbry/wallet/transaction.py b/lbry/wallet/transaction.py index cfd637687..264357f56 100644 --- a/lbry/wallet/transaction.py +++ b/lbry/wallet/transaction.py @@ -159,11 +159,11 @@ class Input(InputOutput): return self.txo_ref.txo.amount @property - def is_my_account(self) -> Optional[bool]: + def is_my_input(self) -> Optional[bool]: """ True if the output this input spends is yours. """ if self.txo_ref.txo is None: return False - return self.txo_ref.txo.is_my_account + return self.txo_ref.txo.is_my_output @classmethod def deserialize_from(cls, stream): @@ -207,7 +207,7 @@ class OutputEffectiveAmountEstimator: class Output(InputOutput): __slots__ = ( - 'amount', 'script', 'is_change', 'is_spent', 'is_received', 'is_my_account', + 'amount', 'script', 'is_internal_transfer', 'is_spent', 'is_my_output', 'is_my_input', 'channel', 'private_key', 'meta', 'purchase', 'purchased_claim', 'purchase_receipt', 'reposted_claim', 'claims', @@ -215,17 +215,17 @@ class Output(InputOutput): def __init__(self, amount: int, script: OutputScript, tx_ref: TXRef = None, position: int = None, - is_change: Optional[bool] = None, is_spent: Optional[bool] = None, - is_received: Optional[bool] = None, is_my_account: Optional[bool] = None, + is_internal_transfer: Optional[bool] = None, is_spent: Optional[bool] = None, + is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None, channel: Optional['Output'] = None, private_key: Optional[str] = None ) -> None: super().__init__(tx_ref, position) self.amount = amount self.script = script - self.is_change = is_change + self.is_internal_transfer = is_internal_transfer self.is_spent = is_spent - self.is_received = is_received - self.is_my_account = is_my_account + self.is_my_output = is_my_output + self.is_my_input = is_my_input self.channel = channel self.private_key = private_key self.purchase: 'Output' = None # txo containing purchase metadata @@ -235,15 +235,17 @@ class Output(InputOutput): self.claims: List['Output'] = None # resolved claims for collection self.meta = {} - def update_annotations(self, annotated): + def update_annotations(self, annotated: 'Output'): if annotated is None: - self.is_change = None + self.is_internal_transfer = None self.is_spent = None - self.is_my_account = None + self.is_my_output = None + self.is_my_input = None else: - self.is_change = annotated.is_change + self.is_internal_transfer = annotated.is_internal_transfer self.is_spent = annotated.is_spent - self.is_my_account = annotated.is_my_account + self.is_my_output = annotated.is_my_output + self.is_my_input = annotated.is_my_input self.channel = annotated.channel if annotated else None self.private_key = annotated.private_key if annotated else None @@ -592,21 +594,21 @@ class Transaction: for txi in self.inputs: if txi.txo_ref.txo is None: continue - if txi.is_my_account is None: - raise ValueError( - "Cannot access net_account_balance if inputs/outputs do not " - "have is_my_account set properly." - ) - if txi.is_my_account: + if txi.is_my_input is True: balance -= txi.amount - for txo in self.outputs: - if txo.is_my_account is None: + elif txi.is_my_input is None: raise ValueError( - "Cannot access net_account_balance if inputs/outputs do not " - "have is_my_account set properly." + "Cannot access net_account_balance if inputs do not " + "have is_my_input set properly." ) - if txo.is_my_account: + for txo in self.outputs: + if txo.is_my_output is True: balance += txo.amount + elif txo.is_my_output is None: + raise ValueError( + "Cannot access net_account_balance if outputs do not " + "have is_my_output set properly." + ) return balance @property @@ -751,7 +753,7 @@ class Transaction: change_hash160 = change_account.ledger.address_to_hash160(change_address) change_amount = change - cost_of_change change_output = Output.pay_pubkey_hash(change_amount, change_hash160) - change_output.is_change = True + change_output.is_internal_transfer = True tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)]) if tx._outputs: @@ -856,17 +858,17 @@ class Transaction: @property def my_inputs(self): for txi in self.inputs: - if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_account: + if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_output: yield txi def _filter_my_outputs(self, f): for txo in self.outputs: - if txo.is_my_account and f(txo.script): + if txo.is_my_output and f(txo.script): yield txo def _filter_other_outputs(self, f): for txo in self.outputs: - if not txo.is_my_account and f(txo.script): + if not txo.is_my_output and f(txo.script): yield txo def _filter_any_outputs(self, f): @@ -898,7 +900,7 @@ class Transaction: def my_abandon_outputs(self): for txi in self.inputs: abandon = txi.txo_ref.txo - if abandon is not None and abandon.is_my_account and abandon.script.is_claim_involved: + if abandon is not None and abandon.is_my_output and abandon.script.is_claim_involved: is_update = False if abandon.script.is_claim_name or abandon.script.is_update_claim: for update in self.my_update_outputs: diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index a99106751..9d33236a1 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -480,25 +480,69 @@ class TransactionOutputCommands(ClaimTestCase): self.assertTrue(r[0]['is_spent']) self.assertTrue(r[1]['is_spent']) - async def test_txo_list_received_filtering(self): + async def test_txo_list_my_input_output_filtering(self): wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True) address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id) - await self.channel_create(claim_address=address2) + await self.channel_create('@kept-channel') + await self.channel_create('@sent-channel', claim_address=address2) - r = await self.txo_list(include_is_received=True) - self.assertEqual(2, len(r)) - self.assertFalse(r[0]['is_received']) - self.assertTrue(r[1]['is_received']) - rt = await self.txo_list(is_not_received=True) - self.assertEqual(1, len(rt)) - self.assertEqual(rt[0], r[0]) - rf = await self.txo_list(is_received=True) - self.assertEqual(1, len(rf)) - self.assertEqual(rf[0], r[1]) + # all txos on second wallet + received_channel, = await self.txo_list(wallet_id=wallet2.id, is_my_input_or_output=True) + self.assertEqual('1.0', received_channel['amount']) + self.assertFalse(received_channel['is_my_input']) + self.assertTrue(received_channel['is_my_output']) + self.assertFalse(received_channel['is_internal_transfer']) - r = await self.txo_list(include_is_received=True, wallet_id=wallet2.id) - self.assertEqual(1, len(r)) - self.assertTrue(r[0]['is_received']) + # all txos on default wallet + r = await self.txo_list(is_my_input_or_output=True) + self.assertEqual( + ['1.0', '7.947786', '1.0', '8.973893', '10.0'], + [t['amount'] for t in r] + ) + + sent_channel, change2, kept_channel, change1, initial_funds = r + + self.assertTrue(sent_channel['is_my_input']) + self.assertFalse(sent_channel['is_my_output']) + self.assertFalse(sent_channel['is_internal_transfer']) + self.assertTrue(change2['is_my_input']) + self.assertTrue(change2['is_my_output']) + self.assertTrue(change2['is_internal_transfer']) + + self.assertTrue(kept_channel['is_my_input']) + self.assertTrue(kept_channel['is_my_output']) + self.assertFalse(kept_channel['is_internal_transfer']) + self.assertTrue(change1['is_my_input']) + self.assertTrue(change1['is_my_output']) + self.assertTrue(change1['is_internal_transfer']) + + self.assertFalse(initial_funds['is_my_input']) + self.assertTrue(initial_funds['is_my_output']) + self.assertFalse(initial_funds['is_internal_transfer']) + + # my stuff and stuff i sent excluding "change" + r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True) + self.assertEqual([sent_channel, kept_channel, initial_funds], r) + + # my unspent stuff and stuff i sent excluding "change" + r = await self.txo_list(is_my_input_or_output=True, unspent=True, exclude_internal_transfers=True) + self.assertEqual([sent_channel, kept_channel], r) + + # only "change" + r = await self.txo_list(is_my_input=True, is_my_output=True, type="other") + self.assertEqual([change2, change1], r) + + # only unspent "change" + r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", unspent=True) + self.assertEqual([change2], r) + + # all my unspent stuff + r = await self.txo_list(is_my_output=True, unspent=True) + self.assertEqual([change2, kept_channel], r) + + # stuff i sent + r = await self.txo_list(is_not_my_output=True) + self.assertEqual([sent_channel], r) class ClaimCommands(ClaimTestCase): @@ -617,7 +661,7 @@ class ClaimCommands(ClaimTestCase): self.assertTrue(r[1]['meta']['is_controlling']) # check that metadata is transfered - self.assertTrue(r[0]['is_mine']) + self.assertTrue(r[0]['is_my_output']) class ChannelCommands(CommandTestCase): diff --git a/tests/unit/wallet/test_database.py b/tests/unit/wallet/test_database.py index 50a40ad91..68d67515e 100644 --- a/tests/unit/wallet/test_database.py +++ b/tests/unit/wallet/test_database.py @@ -355,29 +355,29 @@ class TestQueries(AsyncioTestCase): txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts) self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs]) - self.assertEqual(txs[0].inputs[0].is_my_account, True) - self.assertEqual(txs[0].outputs[0].is_my_account, False) - self.assertEqual(txs[1].inputs[0].is_my_account, False) - self.assertEqual(txs[1].outputs[0].is_my_account, True) + self.assertEqual(txs[0].inputs[0].is_my_input, True) + self.assertEqual(txs[0].outputs[0].is_my_output, False) + self.assertEqual(txs[1].inputs[0].is_my_input, False) + self.assertEqual(txs[1].outputs[0].is_my_output, True) txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2]) self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs]) - self.assertEqual(txs[0].inputs[0].is_my_account, True) - self.assertEqual(txs[0].outputs[0].is_my_account, False) - self.assertEqual(txs[1].inputs[0].is_my_account, False) - self.assertEqual(txs[1].outputs[0].is_my_account, True) + self.assertEqual(txs[0].inputs[0].is_my_input, True) + self.assertEqual(txs[0].outputs[0].is_my_output, False) + self.assertEqual(txs[1].inputs[0].is_my_input, False) + self.assertEqual(txs[1].outputs[0].is_my_output, True) self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account2])) tx = await self.ledger.db.get_transaction(txid=tx2.id) self.assertEqual(tx.id, tx2.id) - self.assertFalse(tx.inputs[0].is_my_account) - self.assertFalse(tx.outputs[0].is_my_account) + self.assertFalse(tx.inputs[0].is_my_input) + self.assertFalse(tx.outputs[0].is_my_output) tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id) - self.assertTrue(tx.inputs[0].is_my_account) - self.assertFalse(tx.outputs[0].is_my_account) + self.assertTrue(tx.inputs[0].is_my_input) + self.assertFalse(tx.outputs[0].is_my_output) tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id) - self.assertFalse(tx.inputs[0].is_my_account) - self.assertTrue(tx.outputs[0].is_my_account) + self.assertFalse(tx.inputs[0].is_my_input) + self.assertTrue(tx.outputs[0].is_my_output) # height 0 sorted to the top with the rest in descending order tx4 = await self.create_tx_from_nothing(account1, 0) diff --git a/tests/unit/wallet/test_transaction.py b/tests/unit/wallet/test_transaction.py index a876a12b9..7c0942cd0 100644 --- a/tests/unit/wallet/test_transaction.py +++ b/tests/unit/wallet/test_transaction.py @@ -82,14 +82,14 @@ class TestSizeAndFeeEstimation(AsyncioTestCase): class TestAccountBalanceImpactFromTransaction(unittest.TestCase): - def test_is_my_account_not_set(self): + def test_is_my_output_not_set(self): tx = get_transaction() with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"): _ = tx.net_account_balance - tx.inputs[0].txo_ref.txo.is_my_account = True + tx.inputs[0].txo_ref.txo.is_my_output = True with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"): _ = tx.net_account_balance - tx.outputs[0].is_my_account = True + tx.outputs[0].is_my_output = True # all inputs/outputs are set now so it should work _ = tx.net_account_balance @@ -98,9 +98,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) - tx.inputs[0].txo_ref.txo.is_my_account = True - tx.outputs[0].is_my_account = False - tx.outputs[1].is_my_account = True + tx.inputs[0].txo_ref.txo.is_my_output = True + tx.outputs[0].is_my_output = False + tx.outputs[1].is_my_output = True self.assertEqual(tx.net_account_balance, -200*CENT) def test_paying_from_other_account_to_my_account(self): @@ -108,9 +108,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) - tx.inputs[0].txo_ref.txo.is_my_account = False - tx.outputs[0].is_my_account = True - tx.outputs[1].is_my_account = False + tx.inputs[0].txo_ref.txo.is_my_output = False + tx.outputs[0].is_my_output = True + tx.outputs[1].is_my_output = False self.assertEqual(tx.net_account_balance, 190*CENT) def test_paying_from_my_account_to_my_account(self): @@ -118,9 +118,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) - tx.inputs[0].txo_ref.txo.is_my_account = True - tx.outputs[0].is_my_account = True - tx.outputs[1].is_my_account = True + tx.inputs[0].txo_ref.txo.is_my_output = True + tx.outputs[0].is_my_output = True + tx.outputs[1].is_my_output = True self.assertEqual(tx.net_account_balance, -10*CENT) # lost to fee From 93fc883b9001224de89b0a00fa6b7ee945e304f0 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 20 Mar 2020 12:09:20 -0400 Subject: [PATCH 03/11] fixing unit tests --- lbry/extras/daemon/daemon.py | 2 +- lbry/wallet/database.py | 2 +- tests/unit/test_cli.py | 2 +- tests/unit/wallet/test_database.py | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 59f1b0750..b727df8ce 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4145,7 +4145,7 @@ class Daemon(metaclass=JSONRPCServerType): --is_my_input : (bool) show outputs created by you --is_not_my_input : (bool) show outputs not created by you --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: - --is_my_input --is_my_output --type=other + "--is_my_input --is_my_output --type=other" this allows to exclude "change" payments, this flag can be used in combination with any of the other flags --account_id= : (str) id of the account to query diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 429c5d1c3..d5de0ef37 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -677,7 +677,7 @@ class Database(SQLiteMixin): self, cols, accounts=None, is_my_input=None, is_my_output=True, is_my_input_or_output=None, exclude_internal_transfers=False, include_is_spent=False, include_is_my_input=False, - read_only=True, **constraints): + read_only=False, **constraints): for rename_col in ('txid', 'txoid'): for rename_constraint in (rename_col, rename_col+'__in', rename_col+'__not_in'): if rename_constraint in constraints: diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 64ad1a3f0..fab143d8c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -218,7 +218,7 @@ class DaemonDocsTests(TestCase): try: docopt.docopt(fn.__doc__, ()) except docopt.DocoptLanguageError as err: - failures.append(f"invalid docstring for {name}, {err.message}") + failures.append(f"invalid docstring for {name}, {err.args[0]}") except docopt.DocoptExit: pass if failures: diff --git a/tests/unit/wallet/test_database.py b/tests/unit/wallet/test_database.py index 68d67515e..7ceed23d7 100644 --- a/tests/unit/wallet/test_database.py +++ b/tests/unit/wallet/test_database.py @@ -353,14 +353,14 @@ class TestQueries(AsyncioTestCase): self.assertListEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs]) self.assertListEqual([3, 2, 1], [tx.height for tx in txs]) - txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts) + txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts, include_is_my_output=True) self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs]) self.assertEqual(txs[0].inputs[0].is_my_input, True) self.assertEqual(txs[0].outputs[0].is_my_output, False) self.assertEqual(txs[1].inputs[0].is_my_input, False) self.assertEqual(txs[1].outputs[0].is_my_output, True) - txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2]) + txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2], include_is_my_output=True) self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs]) self.assertEqual(txs[0].inputs[0].is_my_input, True) self.assertEqual(txs[0].outputs[0].is_my_output, False) @@ -370,20 +370,20 @@ class TestQueries(AsyncioTestCase): tx = await self.ledger.db.get_transaction(txid=tx2.id) self.assertEqual(tx.id, tx2.id) - self.assertFalse(tx.inputs[0].is_my_input) - self.assertFalse(tx.outputs[0].is_my_output) - tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id) + self.assertIsNone(tx.inputs[0].is_my_input) + self.assertIsNone(tx.outputs[0].is_my_output) + tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id, include_is_my_output=True) self.assertTrue(tx.inputs[0].is_my_input) self.assertFalse(tx.outputs[0].is_my_output) - tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id) + tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id, include_is_my_output=True) self.assertFalse(tx.inputs[0].is_my_input) self.assertTrue(tx.outputs[0].is_my_output) # height 0 sorted to the top with the rest in descending order tx4 = await self.create_tx_from_nothing(account1, 0) txos = await self.ledger.db.get_txos() - self.assertListEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos]) - self.assertListEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos]) + self.assertListEqual([0, 3, 2, 2, 1], [txo.tx_ref.height for txo in txos]) + self.assertListEqual([tx4.id, tx3.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos]) txs = await self.ledger.db.get_transactions(accounts=[account1, account2]) self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs]) self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs]) From d6d83a5c76a567016d4b72c104e370607124f8b0 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 20 Mar 2020 18:24:24 -0400 Subject: [PATCH 04/11] integration test fix --- lbry/wallet/account.py | 17 ++++++----- lbry/wallet/database.py | 28 +++++++++++-------- .../blockchain/test_transactions.py | 6 ++-- .../blockchain/test_wallet_commands.py | 25 ++++++++++------- 4 files changed, 43 insertions(+), 33 deletions(-) diff --git a/lbry/wallet/account.py b/lbry/wallet/account.py index 5194210d5..c37457c5d 100644 --- a/lbry/wallet/account.py +++ b/lbry/wallet/account.py @@ -454,7 +454,7 @@ class Account: def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints): if not include_claims: - constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])}) + constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])}) if confirmations > 0: height = self.ledger.headers.height - (confirmations-1) constraints.update({'height__lte': height, 'height__gt': 0}) @@ -569,14 +569,13 @@ class Account: total = await get_total_balance() if reserved_subtotals: claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES) - for amount, spent, from_me, to_me, height in await self.get_support_summary(): - if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1): + for txo in await self.get_support_summary(): + if confirmations > 0 and not 0 < txo.tx_ref.height <= self.ledger.headers.height - (confirmations - 1): continue - if not spent and to_me: - if from_me: - supports_balance += amount - else: - tips_balance += amount + if txo.is_my_input: + supports_balance += txo.amount + else: + tips_balance += txo.amount reserved = claims_balance + supports_balance + tips_balance else: reserved = await self.get_balance( @@ -634,7 +633,7 @@ class Account: return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints) def get_support_summary(self): - return self.ledger.db.get_supports_summary(account_id=self.id) + return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self]) async def release_all_outputs(self): await self.ledger.db.release_all_outputs(self) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index d5de0ef37..bb301ab29 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -839,7 +839,7 @@ class Database(SQLiteMixin): return txos - async def get_txo_count(self, unspent=False, **constraints): + def _clean_txo_constraints_for_aggregation(self, unspent, constraints): constraints.pop('include_is_my_input', None) constraints.pop('include_is_my_output', None) constraints.pop('wallet', None) @@ -849,9 +849,17 @@ class Database(SQLiteMixin): constraints.pop('order_by', None) if unspent: self.constrain_unspent(constraints) + + async def get_txo_count(self, unspent=False, **constraints): + self._clean_txo_constraints_for_aggregation(unspent, constraints) count = await self.select_txos('COUNT(*) as total', **constraints) return count[0]['total'] + async def get_txo_sum(self, unspent=False, **constraints): + self._clean_txo_constraints_for_aggregation(unspent, constraints) + result = await self.select_txos('SUM(amount) as total', **constraints) + return result[0]['total'] + def get_utxos(self, read_only=False, **constraints): return self.get_txos(unspent=True, read_only=read_only, **constraints) @@ -1019,13 +1027,11 @@ class Database(SQLiteMixin): " )", (account.public_key.address, ) ) - def get_supports_summary(self, account_id, read_only=False): - return self.db.execute_fetchall(f""" - select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent, - (txo.txid in - (select txi.txid from txi join account_address a on txi.address = a.address - where a.account = ?)) as from_me, - (txo.address in (select address from account_address where account=?)) as to_me, - tx.height - from txo join tx using (txid) where txo_type={TXO_TYPES['support']} - """, (account_id, account_id), read_only=read_only) + def get_supports_summary(self, read_only=False, **constraints): + return self.get_txos( + txo_type=TXO_TYPES['support'], + unspent=True, is_my_output=True, + include_is_my_input=True, + no_tx=True, read_only=read_only, + **constraints + ) diff --git a/tests/integration/blockchain/test_transactions.py b/tests/integration/blockchain/test_transactions.py index 82aa9c16a..6a0ace201 100644 --- a/tests/integration/blockchain/test_transactions.py +++ b/tests/integration/blockchain/test_transactions.py @@ -122,13 +122,13 @@ class BasicTransactionTests(IntegrationTestCase): await self.blockchain.generate(1) await self.ledger.wait(tx) # confirmed - tx = (await account1.get_transactions())[1] + tx = (await account1.get_transactions(include_is_my_input=True, include_is_my_output=True))[1] self.assertEqual(satoshis_to_coins(tx.inputs[0].amount), '1.1') self.assertEqual(satoshis_to_coins(tx.inputs[1].amount), '1.1') self.assertEqual(satoshis_to_coins(tx.outputs[0].amount), '2.0') self.assertEqual(tx.outputs[0].get_address(self.ledger), address2) - self.assertFalse(tx.outputs[0].is_change) - self.assertTrue(tx.outputs[1].is_change) + self.assertTrue(tx.outputs[0].is_internal_transfer) + self.assertTrue(tx.outputs[1].is_internal_transfer) async def test_history_edge_cases(self): await self.blockchain.generate(300) diff --git a/tests/integration/blockchain/test_wallet_commands.py b/tests/integration/blockchain/test_wallet_commands.py index 8500efa3c..c65fdfb10 100644 --- a/tests/integration/blockchain/test_wallet_commands.py +++ b/tests/integration/blockchain/test_wallet_commands.py @@ -82,6 +82,11 @@ class WalletCommands(CommandTestCase): async def test_granular_balances(self): account2 = await self.daemon.jsonrpc_account_create("Tip-er") + wallet2 = await self.daemon.jsonrpc_wallet_create('foo', create_account=True) + account3 = wallet2.default_account + address3 = await self.daemon.jsonrpc_address_unused(account3.id, wallet2.id) + await self.confirm_tx(await self.blockchain.send_to_address(address3, 1)) + await self.generate(1) account_balance = self.daemon.jsonrpc_account_balance wallet_balance = self.daemon.jsonrpc_wallet_balance @@ -128,7 +133,7 @@ class WalletCommands(CommandTestCase): # tip received support1 = await self.support_create( - self.get_claim_id(stream1), '0.3', tip=True, funding_account_ids=[account2.id] + self.get_claim_id(stream1), '0.3', tip=True, wallet_id=wallet2.id ) self.assertEqual(await account_balance(), { 'total': '9.27741', @@ -137,8 +142,8 @@ class WalletCommands(CommandTestCase): 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'} }) self.assertEqual(await wallet_balance(), { - 'total': '9.977268', - 'available': '6.677268', + 'total': '10.27741', + 'available': '6.97741', 'reserved': '3.3', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'} }) @@ -153,8 +158,8 @@ class WalletCommands(CommandTestCase): 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) self.assertEqual(await wallet_balance(), { - 'total': '9.977161', - 'available': '6.977161', + 'total': '10.277303', + 'available': '7.277303', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) @@ -165,17 +170,17 @@ class WalletCommands(CommandTestCase): # tip another claim await self.support_create( - self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id] + self.get_claim_id(stream2), '0.2', tip=True, wallet_id=wallet2.id ) self.assertEqual(await account_balance(), { - 'total': '9.077157', - 'available': '6.077157', + 'total': '9.277303', + 'available': '6.277303', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) self.assertEqual(await wallet_balance(), { - 'total': '9.938908', - 'available': '6.638908', + 'total': '10.439196', + 'available': '7.139196', 'reserved': '3.3', 'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'} }) From 6293e227ea5a30516e1cc14a297595ba34e55fb3 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 20 Mar 2020 19:07:16 -0400 Subject: [PATCH 05/11] added txo_sum command --- lbry/extras/daemon/daemon.py | 92 ++++++++++++++----- lbry/testcase.py | 3 + lbry/wallet/ledger.py | 3 + .../blockchain/test_claim_commands.py | 17 +++- 4 files changed, 90 insertions(+), 25 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index b727df8ce..e0523367f 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4106,17 +4106,37 @@ class Daemon(metaclass=JSONRPCServerType): return self.wallet_manager.get_transaction(txid) TXO_DOC = """ - List transaction outputs. + List and sum transaction outputs. """ - @requires(WALLET_COMPONENT) - def jsonrpc_txo_list( - self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin + @staticmethod + def _constrain_txo_from_kwargs( + constraints, type=None, txid=None, # pylint: disable=redefined-builtin claim_id=None, name=None, unspent=False, is_my_input_or_output=None, exclude_internal_transfers=False, is_my_output=None, is_not_my_output=None, - is_my_input=None, is_not_my_input=None, - wallet_id=None, page=None, page_size=None, resolve=False): + is_my_input=None, is_not_my_input=None): + constraints['unspent'] = unspent + constraints['exclude_internal_transfers'] = exclude_internal_transfers + if is_my_input_or_output is True: + constraints['is_my_input_or_output'] = True + else: + if is_my_input is True: + constraints['is_my_input'] = True + elif is_not_my_input is True: + constraints['is_my_input'] = False + if is_my_output is True: + constraints['is_my_output'] = True + elif is_not_my_output is True: + constraints['is_my_output'] = False + database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) + database.constrain_single_or_list(constraints, 'claim_id', claim_id) + database.constrain_single_or_list(constraints, 'claim_name', name) + database.constrain_single_or_list(constraints, 'txid', txid) + return constraints + + @requires(WALLET_COMPONENT) + def jsonrpc_txo_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False, **kwargs): """ List my transaction outputs. @@ -4166,28 +4186,56 @@ class Daemon(metaclass=JSONRPCServerType): claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts, read_only=True) constraints = { 'resolve': resolve, - 'unspent': unspent, - 'exclude_internal_transfers': exclude_internal_transfers, 'include_is_spent': True, 'include_is_my_input': True, 'include_is_my_output': True, } - if is_my_input_or_output is True: - constraints['is_my_input_or_output'] = True - else: - if is_my_input is True: - constraints['is_my_input'] = True - elif is_not_my_input is True: - constraints['is_my_input'] = False - if is_my_output is True: - constraints['is_my_output'] = True - elif is_not_my_output is True: - constraints['is_my_output'] = False - database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) - database.constrain_single_or_list(constraints, 'claim_id', claim_id) - database.constrain_single_or_list(constraints, 'claim_name', name) + self._constrain_txo_from_kwargs(constraints, **kwargs) return paginate_rows(claims, claim_count, page, page_size, **constraints) + @requires(WALLET_COMPONENT) + def jsonrpc_txo_sum(self, account_id=None, wallet_id=None, **kwargs): + """ + Sum transaction outputs. + + Usage: + txo_list [--account_id=] [--type=...] [--txid=...] + [--claim_id=...] [--name=...] [--unspent] + [--is_my_input_or_output | + [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] + ] + [--exclude_internal_transfers] + [--wallet_id=] + + Options: + --type= : (str or list) claim type: stream, channel, support, + purchase, collection, repost, other + --txid= : (str or list) transaction id of outputs + --claim_id= : (str or list) claim id + --name= : (str or list) claim name + --unspent : (bool) hide spent outputs, show only unspent ones + --is_my_input_or_output : (bool) txos which have your inputs or your outputs, + if using this flag the other related flags + are ignored (--is_my_output, --is_my_input, etc) + --is_my_output : (bool) show outputs controlled by you + --is_not_my_output : (bool) show outputs not controlled by you + --is_my_input : (bool) show outputs created by you + --is_not_my_input : (bool) show outputs not created by you + --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: + "--is_my_input --is_my_output --type=other" + this allows to exclude "change" payments, this + flag can be used in combination with any of the other flags + --account_id= : (str) id of the account to query + --wallet_id= : (str) restrict results to specific wallet + + Returns: {Paginated[Output]} + """ + wallet = self.wallet_manager.get_wallet_or_default(wallet_id) + return self.ledger.get_txo_sum( + wallet=wallet, accounts=[wallet.get_account_or_error(account_id)] if account_id else wallet.accounts, + read_only=True, **self._constrain_txo_from_kwargs({}, **kwargs) + ) + UTXO_DOC = """ Unspent transaction management. """ diff --git a/lbry/testcase.py b/lbry/testcase.py index 483511ae9..798937db6 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -572,6 +572,9 @@ class CommandTestCase(IntegrationTestCase): async def txo_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items'] + async def txo_sum(self, *args, **kwargs): + return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs)) + async def claim_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items'] diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 5c1f8d902..7b8eefdab 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -274,6 +274,9 @@ class Ledger(metaclass=LedgerRegistry): def get_txo_count(self, **constraints): return self.db.get_txo_count(**constraints) + def get_txo_sum(self, **constraints): + return self.db.get_txo_sum(**constraints) + def get_transactions(self, **constraints): return self.db.get_transactions(**constraints) diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index 9d33236a1..252d8299b 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -9,6 +9,7 @@ from lbry.error import InsufficientFundsError from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE from lbry.testcase import CommandTestCase from lbry.wallet.transaction import Transaction +from lbry.wallet.util import satoshis_to_coins as lbc log = logging.getLogger(__name__) @@ -420,11 +421,21 @@ class TransactionCommands(ClaimTestCase): class TransactionOutputCommands(ClaimTestCase): - async def test_txo_list_filtering(self): + async def test_txo_list_and_sum_filtering(self): channel_id = self.get_claim_id(await self.channel_create()) + self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', unspent=True))) await self.channel_update(channel_id, bid='0.5') - stream_id = self.get_claim_id(await self.stream_create()) - await self.stream_update(stream_id, bid='0.5') + self.assertEqual('0.5', lbc(await self.txo_sum(type='channel', unspent=True))) + self.assertEqual('1.5', lbc(await self.txo_sum(type='channel'))) + + stream_id = self.get_claim_id(await self.stream_create(bid='1.3')) + self.assertEqual('1.3', lbc(await self.txo_sum(type='stream', unspent=True))) + await self.stream_update(stream_id, bid='0.7') + self.assertEqual('0.7', lbc(await self.txo_sum(type='stream', unspent=True))) + self.assertEqual('2.0', lbc(await self.txo_sum(type='stream'))) + + self.assertEqual('1.2', lbc(await self.txo_sum(type=['stream', 'channel'], unspent=True))) + self.assertEqual('3.5', lbc(await self.txo_sum(type=['stream', 'channel']))) # type filtering r = await self.txo_list(type='channel') From 5e0324cc91a81fd0e47b8fc42b7196f695c14abd Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 20 Mar 2020 20:22:57 -0400 Subject: [PATCH 06/11] added --reposted_claim_id to txo_list --- lbry/extras/daemon/daemon.py | 10 +++++----- lbry/wallet/database.py | 10 ++++++++-- tests/integration/blockchain/test_claim_commands.py | 4 ++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index e0523367f..0d3359494 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4112,7 +4112,7 @@ class Daemon(metaclass=JSONRPCServerType): @staticmethod def _constrain_txo_from_kwargs( constraints, type=None, txid=None, # pylint: disable=redefined-builtin - claim_id=None, name=None, unspent=False, + claim_id=None, name=None, unspent=False, reposted_claim_id=None, is_my_input_or_output=None, exclude_internal_transfers=False, is_my_output=None, is_not_my_output=None, is_my_input=None, is_not_my_input=None): @@ -4133,6 +4133,7 @@ class Daemon(metaclass=JSONRPCServerType): database.constrain_single_or_list(constraints, 'claim_id', claim_id) database.constrain_single_or_list(constraints, 'claim_name', name) database.constrain_single_or_list(constraints, 'txid', txid) + database.constrain_single_or_list(constraints, 'reposted_claim_id', reposted_claim_id) return constraints @requires(WALLET_COMPONENT) @@ -4196,7 +4197,7 @@ class Daemon(metaclass=JSONRPCServerType): @requires(WALLET_COMPONENT) def jsonrpc_txo_sum(self, account_id=None, wallet_id=None, **kwargs): """ - Sum transaction outputs. + Sum of transaction outputs. Usage: txo_list [--account_id=] [--type=...] [--txid=...] @@ -4204,8 +4205,7 @@ class Daemon(metaclass=JSONRPCServerType): [--is_my_input_or_output | [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] ] - [--exclude_internal_transfers] - [--wallet_id=] + [--exclude_internal_transfers] [--wallet_id=] Options: --type= : (str or list) claim type: stream, channel, support, @@ -4228,7 +4228,7 @@ class Daemon(metaclass=JSONRPCServerType): --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet - Returns: {Paginated[Output]} + Returns: int """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) return self.ledger.get_txo_sum( diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index bb301ab29..5c707be9f 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -439,13 +439,16 @@ class Database(SQLiteMixin): txo_type integer not null default 0, claim_id text, - claim_name text + claim_name text, + + reposted_claim_id text ); create index if not exists txo_txid_idx on txo (txid); create index if not exists txo_address_idx on txo (address); create index if not exists txo_claim_id_idx on txo (claim_id); create index if not exists txo_claim_name_idx on txo (claim_name); create index if not exists txo_txo_type_idx on txo (txo_type); + create index if not exists txo_reposted_claim_idx on txo (reposted_claim_id); """ CREATE_TXI_TABLE = """ @@ -483,7 +486,10 @@ class Database(SQLiteMixin): } if txo.is_claim: if txo.can_decode_claim: - row['txo_type'] = TXO_TYPES.get(txo.claim.claim_type, TXO_TYPES['stream']) + claim = txo.claim + row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream']) + if claim.is_repost: + row['reposted_claim_id'] = claim.repost.reference.claim_id else: row['txo_type'] = TXO_TYPES['stream'] elif txo.is_support: diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index 252d8299b..d7e32301b 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -1023,12 +1023,16 @@ class StreamCommands(ClaimTestCase): claim_id = self.get_claim_id(tx) self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0) + self.assertItemCount(await self.daemon.jsonrpc_txo_list(reposted_claim_id=claim_id), 0) + self.assertItemCount(await self.daemon.jsonrpc_txo_list(type='repost'), 0) tx = await self.stream_repost(claim_id, 'newstuff-again', '1.1') repost_id = self.get_claim_id(tx) self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 1) self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1) self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id) + self.assertEqual((await self.txo_list(reposted_claim_id=claim_id))[0]['claim_id'], repost_id) + self.assertEqual((await self.txo_list(type='repost'))[0]['claim_id'], repost_id) # tags are inherited (non-common / indexed tags) self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2) From 7cb530c33495531a608a95ee9b44306725e7f61e Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 20 Mar 2020 23:19:26 -0400 Subject: [PATCH 07/11] added --channel_id and --order_by to txo_list --- lbry/extras/daemon/daemon.py | 25 ++++++++++++++----- lbry/wallet/database.py | 6 ++++- .../blockchain/test_claim_commands.py | 16 ++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 0d3359494..7645024f4 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -2176,19 +2176,21 @@ class Daemon(metaclass=JSONRPCServerType): Usage: claim_list [--claim_type=...] [--claim_id=...] [--name=...] - [--account_id=] [--wallet_id=] + [--channel_id=...] [--account_id=] [--wallet_id=] [--page=] [--page_size=] - [--resolve] + [--resolve] [--order_by=] Options: --claim_type= : (str or list) claim type: channel, stream, repost, collection --claim_id= : (str or list) claim id + --channel_id= : (str or list) streams in this channel --name= : (str or list) claim name --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata + --order_by= : (str) field to order by: 'name', 'height', 'amount' Returns: {Paginated[Output]} """ @@ -4112,7 +4114,7 @@ class Daemon(metaclass=JSONRPCServerType): @staticmethod def _constrain_txo_from_kwargs( constraints, type=None, txid=None, # pylint: disable=redefined-builtin - claim_id=None, name=None, unspent=False, reposted_claim_id=None, + claim_id=None, channel_id=None, name=None, unspent=False, reposted_claim_id=None, is_my_input_or_output=None, exclude_internal_transfers=False, is_my_output=None, is_not_my_output=None, is_my_input=None, is_not_my_input=None): @@ -4130,6 +4132,7 @@ class Daemon(metaclass=JSONRPCServerType): elif is_not_my_output is True: constraints['is_my_output'] = False database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) + database.constrain_single_or_list(constraints, 'channel_id', channel_id) database.constrain_single_or_list(constraints, 'claim_id', claim_id) database.constrain_single_or_list(constraints, 'claim_name', name) database.constrain_single_or_list(constraints, 'txid', txid) @@ -4137,13 +4140,14 @@ class Daemon(metaclass=JSONRPCServerType): return constraints @requires(WALLET_COMPONENT) - def jsonrpc_txo_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False, **kwargs): + def jsonrpc_txo_list( + self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False, order_by=None, **kwargs): """ List my transaction outputs. Usage: - txo_list [--account_id=] [--type=...] [--txid=...] - [--claim_id=...] [--name=...] [--unspent] + txo_list [--account_id=] [--type=...] [--txid=...] [--unspent] + [--claim_id=...] [--channel_id=...] [--name=...] [--is_my_input_or_output | [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] ] @@ -4156,6 +4160,7 @@ class Daemon(metaclass=JSONRPCServerType): purchase, collection, repost, other --txid= : (str or list) transaction id of outputs --claim_id= : (str or list) claim id + --channel_id= : (str or list) claims in this channel --name= : (str or list) claim name --unspent : (bool) hide spent outputs, show only unspent ones --is_my_input_or_output : (bool) txos which have your inputs or your outputs, @@ -4174,6 +4179,7 @@ class Daemon(metaclass=JSONRPCServerType): --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata + --order_by= : (str) field to order by: 'name', 'height', 'amount' Returns: {Paginated[Output]} """ @@ -4191,6 +4197,13 @@ class Daemon(metaclass=JSONRPCServerType): 'include_is_my_input': True, 'include_is_my_output': True, } + if order_by is not None: + if order_by == 'name': + constraints['order_by'] = 'txo.claim_name' + elif order_by in ('height', 'amount'): + constraints['order_by'] = order_by + else: + raise ValueError(f"'{order_by}' is not a valid --order_by value.") self._constrain_txo_from_kwargs(constraints, **kwargs) return paginate_rows(claims, claim_count, page, page_size, **constraints) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 5c707be9f..571335fee 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -441,6 +441,7 @@ class Database(SQLiteMixin): claim_id text, claim_name text, + channel_id text, reposted_claim_id text ); create index if not exists txo_txid_idx on txo (txid); @@ -448,6 +449,7 @@ class Database(SQLiteMixin): create index if not exists txo_claim_id_idx on txo (claim_id); create index if not exists txo_claim_name_idx on txo (claim_name); create index if not exists txo_txo_type_idx on txo (txo_type); + create index if not exists txo_channel_id_idx on txo (channel_id); create index if not exists txo_reposted_claim_idx on txo (reposted_claim_id); """ @@ -490,6 +492,8 @@ class Database(SQLiteMixin): row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream']) if claim.is_repost: row['reposted_claim_id'] = claim.repost.reference.claim_id + if claim.is_signed: + row['channel_id'] = claim.signing_channel_id else: row['txo_type'] = TXO_TYPES['stream'] elif txo.is_support: @@ -779,7 +783,7 @@ class Database(SQLiteMixin): if include_is_spent: select_columns.append("spent.txoid IS NOT NULL AS is_spent") - if 'order_by' not in constraints: + if 'order_by' not in constraints or constraints['order_by'] == 'height': constraints['order_by'] = [ "tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position" ] diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index d7e32301b..7e3729cbc 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -674,6 +674,22 @@ class ClaimCommands(ClaimTestCase): # check that metadata is transfered self.assertTrue(r[0]['is_my_output']) + async def assertClaimList(self, claim_ids, **kwargs): + self.assertEqual(claim_ids, [c['claim_id'] for c in await self.claim_list(**kwargs)]) + + async def test_list_streams_in_channel_and_order_by(self): + channel1_id = self.get_claim_id(await self.channel_create('@chan-one')) + channel2_id = self.get_claim_id(await self.channel_create('@chan-two')) + stream1_id = self.get_claim_id(await self.stream_create('stream-a', bid='0.3', channel_id=channel1_id)) + stream2_id = self.get_claim_id(await self.stream_create('stream-b', bid='0.9', channel_id=channel1_id)) + stream3_id = self.get_claim_id(await self.stream_create('stream-c', bid='0.6', channel_id=channel2_id)) + await self.assertClaimList([stream2_id, stream1_id], channel_id=channel1_id) + await self.assertClaimList([stream3_id], channel_id=channel2_id) + await self.assertClaimList([stream3_id, stream2_id, stream1_id], channel_id=[channel1_id, channel2_id]) + await self.assertClaimList([stream1_id, stream2_id, stream3_id], claim_type='stream', order_by='name') + await self.assertClaimList([stream1_id, stream3_id, stream2_id], claim_type='stream', order_by='amount') + await self.assertClaimList([stream3_id, stream2_id, stream1_id], claim_type='stream', order_by='height') + class ChannelCommands(CommandTestCase): From 5cd7e9a9b8c479ba06ec80e6df36cc10813adf97 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sat, 21 Mar 2020 15:08:14 -0400 Subject: [PATCH 08/11] increment scema version and force specific indexes to be used for get_txos() --- lbry/wallet/database.py | 6 +++--- lbry/wallet/ledger.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 571335fee..abae15d07 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -387,7 +387,7 @@ def dict_row_factory(cursor, row): class Database(SQLiteMixin): - SCHEMA_VERSION = "1.1" + SCHEMA_VERSION = "1.2" PRAGMAS = """ pragma journal_mode=WAL; @@ -633,7 +633,7 @@ class Database(SQLiteMixin): txo.id: txo for txo in (await self.get_txos( wallet=wallet, - txid__in=txids[offset:offset+step], + txid__in=txids[offset:offset+step], order_by='txo.txid', include_is_spent=include_is_spent, include_is_my_input=include_is_my_input, include_is_my_output=include_is_my_output, @@ -646,7 +646,7 @@ class Database(SQLiteMixin): txo.id: txo for txo in (await self.get_txos( wallet=wallet, - txoid__in=txi_txoids[offset:offset+step], + txoid__in=txi_txoids[offset:offset+step], order_by='txo.txoid', include_is_my_output=include_is_my_output, )) }) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 7b8eefdab..639446d3f 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -524,7 +524,9 @@ class Ledger(metaclass=LedgerRegistry): check_db_for_txos.append(txi.txo_ref.id) referenced_txos = {} if not check_db_for_txos else { - txo.id: txo for txo in await self.db.get_txos(txoid__in=check_db_for_txos, no_tx=True) + txo.id: txo for txo in await self.db.get_txos( + txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True + ) } for txi in tx.inputs: From 15091052beb9c947686cbfe283d1b4767a41290d Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sat, 21 Mar 2020 18:06:05 -0400 Subject: [PATCH 09/11] added --no_totals to txo_list --- lbry/extras/daemon/daemon.py | 43 +++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 7645024f4..79c30c1fa 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -141,7 +141,7 @@ def encode_pagination_doc(items): } -async def paginate_rows(get_records: Callable, get_record_count: Callable, +async def paginate_rows(get_records: Callable, get_record_count: Optional[Callable], page: Optional[int], page_size: Optional[int], **constraints): page = max(1, page or 1) page_size = max(1, page_size or DEFAULT_PAGE_SIZE) @@ -150,13 +150,12 @@ async def paginate_rows(get_records: Callable, get_record_count: Callable, "limit": page_size }) items = await get_records(**constraints) - total_items = await get_record_count(**constraints) - return { - "items": items, - "total_pages": int((total_items + (page_size - 1)) / page_size), - "total_items": total_items, - "page": page, "page_size": page_size - } + result = {"items": items, "page": page, "page_size": page_size} + if get_record_count is not None: + total_items = await get_record_count(**constraints) + result["total_pages"] = int((total_items + (page_size - 1)) / page_size) + result["total_items"] = total_items + return result def paginate_list(items: List, page: Optional[int], page_size: Optional[int]): @@ -2178,7 +2177,7 @@ class Daemon(metaclass=JSONRPCServerType): claim_list [--claim_type=...] [--claim_id=...] [--name=...] [--channel_id=...] [--account_id=] [--wallet_id=] [--page=] [--page_size=] - [--resolve] [--order_by=] + [--resolve] [--order_by=] [--no_totals] Options: --claim_type= : (str or list) claim type: channel, stream, repost, collection @@ -2191,6 +2190,8 @@ class Daemon(metaclass=JSONRPCServerType): --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata --order_by= : (str) field to order by: 'name', 'height', 'amount' + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -2706,7 +2707,7 @@ class Daemon(metaclass=JSONRPCServerType): Usage: channel_list [ | --account_id=] [--wallet_id=] [--name=...] [--claim_id=...] - [--page=] [--page_size=] [--resolve] + [--page=] [--page_size=] [--resolve] [--no_totals] Options: --name= : (str or list) channel name @@ -2716,6 +2717,8 @@ class Daemon(metaclass=JSONRPCServerType): --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each channel to provide additional metadata + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -3455,7 +3458,7 @@ class Daemon(metaclass=JSONRPCServerType): Usage: stream_list [ | --account_id=] [--wallet_id=] [--name=...] [--claim_id=...] - [--page=] [--page_size=] [--resolve] + [--page=] [--page_size=] [--resolve] [--no_totals] Options: --name= : (str or list) stream name @@ -3465,6 +3468,8 @@ class Daemon(metaclass=JSONRPCServerType): --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each stream to provide additional metadata + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -3923,7 +3928,7 @@ class Daemon(metaclass=JSONRPCServerType): Usage: support_list [ | --account_id=] [--wallet_id=] [--name=...] [--claim_id=...] [--tips] - [--page=] [--page_size=] + [--page=] [--page_size=] [--no_totals] Options: --name= : (str or list) claim name @@ -3933,13 +3938,18 @@ class Daemon(metaclass=JSONRPCServerType): --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ kwargs['type'] = 'support' kwargs['unspent'] = True if tips is True: + kwargs['is_not_my_input'] = True kwargs['is_my_output'] = True + else: + kwargs['is_my_input_or_output'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) @@ -4141,7 +4151,8 @@ class Daemon(metaclass=JSONRPCServerType): @requires(WALLET_COMPONENT) def jsonrpc_txo_list( - self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False, order_by=None, **kwargs): + self, account_id=None, wallet_id=None, page=None, page_size=None, + resolve=False, order_by=None, no_totals=False, **kwargs): """ List my transaction outputs. @@ -4153,7 +4164,7 @@ class Daemon(metaclass=JSONRPCServerType): ] [--exclude_internal_transfers] [--wallet_id=] [--page=] [--page_size=] - [--resolve] + [--resolve] [--no_totals] Options: --type= : (str or list) claim type: stream, channel, support, @@ -4180,6 +4191,8 @@ class Daemon(metaclass=JSONRPCServerType): --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata --order_by= : (str) field to order by: 'name', 'height', 'amount' + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -4205,7 +4218,7 @@ class Daemon(metaclass=JSONRPCServerType): else: raise ValueError(f"'{order_by}' is not a valid --order_by value.") self._constrain_txo_from_kwargs(constraints, **kwargs) - return paginate_rows(claims, claim_count, page, page_size, **constraints) + return paginate_rows(claims, None if no_totals else claim_count, page, page_size, **constraints) @requires(WALLET_COMPONENT) def jsonrpc_txo_sum(self, account_id=None, wallet_id=None, **kwargs): From 6a58148a89e4b1a844f4e85b9605e39bc317fc02 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sat, 21 Mar 2020 18:16:25 -0400 Subject: [PATCH 10/11] added support for --order_by=none --- lbry/extras/daemon/daemon.py | 6 +++--- lbry/wallet/database.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 79c30c1fa..e8ff30a6f 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4164,7 +4164,7 @@ class Daemon(metaclass=JSONRPCServerType): ] [--exclude_internal_transfers] [--wallet_id=] [--page=] [--page_size=] - [--resolve] [--no_totals] + [--resolve] [--order_by=][--no_totals] Options: --type= : (str or list) claim type: stream, channel, support, @@ -4190,7 +4190,7 @@ class Daemon(metaclass=JSONRPCServerType): --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata - --order_by= : (str) field to order by: 'name', 'height', 'amount' + --order_by= : (str) field to order by: 'name', 'height', 'amount' and 'none' --no_totals : (bool) do not calculate the total number of pages and items in result set (significant performance boost) @@ -4213,7 +4213,7 @@ class Daemon(metaclass=JSONRPCServerType): if order_by is not None: if order_by == 'name': constraints['order_by'] = 'txo.claim_name' - elif order_by in ('height', 'amount'): + elif order_by in ('height', 'amount', 'none'): constraints['order_by'] = order_by else: raise ValueError(f"'{order_by}' is not a valid --order_by value.") diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index abae15d07..d8265459e 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -787,6 +787,8 @@ class Database(SQLiteMixin): constraints['order_by'] = [ "tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position" ] + elif constraints.get('order_by', None) == 'none': + del constraints['order_by'] rows = await self.select_txos(', '.join(select_columns), read_only=read_only, **constraints) From 87089b8e832b8ce9bc7102fe120c6bedd5572c60 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sat, 21 Mar 2020 18:48:06 -0400 Subject: [PATCH 11/11] fix support test --- lbry/extras/daemon/daemon.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index e8ff30a6f..cb3fbf2d9 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -3947,9 +3947,6 @@ class Daemon(metaclass=JSONRPCServerType): kwargs['unspent'] = True if tips is True: kwargs['is_not_my_input'] = True - kwargs['is_my_output'] = True - else: - kwargs['is_my_input_or_output'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT)