diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 51507a526..cb3fbf2d9 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -141,7 +141,7 @@ def encode_pagination_doc(items): } -async def paginate_rows(get_records: Callable, get_record_count: Callable, +async def paginate_rows(get_records: Callable, get_record_count: Optional[Callable], page: Optional[int], page_size: Optional[int], **constraints): page = max(1, page or 1) page_size = max(1, page_size or DEFAULT_PAGE_SIZE) @@ -150,13 +150,12 @@ async def paginate_rows(get_records: Callable, get_record_count: Callable, "limit": page_size }) items = await get_records(**constraints) - total_items = await get_record_count(**constraints) - return { - "items": items, - "total_pages": int((total_items + (page_size - 1)) / page_size), - "total_items": total_items, - "page": page, "page_size": page_size - } + result = {"items": items, "page": page, "page_size": page_size} + if get_record_count is not None: + total_items = await get_record_count(**constraints) + result["total_pages"] = int((total_items + (page_size - 1)) / page_size) + result["total_items"] = total_items + return result def paginate_list(items: List, page: Optional[int], page_size: Optional[int]): @@ -2176,19 +2175,23 @@ class Daemon(metaclass=JSONRPCServerType): Usage: claim_list [--claim_type=...] [--claim_id=...] [--name=...] - [--account_id=] [--wallet_id=] + [--channel_id=...] [--account_id=] [--wallet_id=] [--page=] [--page_size=] - [--resolve] + [--resolve] [--order_by=] [--no_totals] Options: --claim_type= : (str or list) claim type: channel, stream, repost, collection --claim_id= : (str or list) claim id + --channel_id= : (str or list) streams in this channel --name= : (str or list) claim name --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata + --order_by= : (str) field to order by: 'name', 'height', 'amount' + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -2704,7 +2707,7 @@ class Daemon(metaclass=JSONRPCServerType): Usage: channel_list [ | --account_id=] [--wallet_id=] [--name=...] [--claim_id=...] - [--page=] [--page_size=] [--resolve] + [--page=] [--page_size=] [--resolve] [--no_totals] Options: --name= : (str or list) channel name @@ -2714,6 +2717,8 @@ class Daemon(metaclass=JSONRPCServerType): --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each channel to provide additional metadata + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -3453,7 +3458,7 @@ class Daemon(metaclass=JSONRPCServerType): Usage: stream_list [ | --account_id=] [--wallet_id=] [--name=...] [--claim_id=...] - [--page=] [--page_size=] [--resolve] + [--page=] [--page_size=] [--resolve] [--no_totals] Options: --name= : (str or list) stream name @@ -3463,6 +3468,8 @@ class Daemon(metaclass=JSONRPCServerType): --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each stream to provide additional metadata + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -3921,24 +3928,25 @@ class Daemon(metaclass=JSONRPCServerType): Usage: support_list [ | --account_id=] [--wallet_id=] [--name=...] [--claim_id=...] [--tips] - [--page=] [--page_size=] + [--page=] [--page_size=] [--no_totals] Options: --name= : (str or list) claim name --claim_id= : (str or list) claim id - --tips : (bool) only show tips (is_received=true) + --tips : (bool) only show tips --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ kwargs['type'] = 'support' kwargs['unspent'] = True - kwargs['include_is_received'] = True if tips is True: - kwargs['is_received'] = True + kwargs['is_not_my_input'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) @@ -4107,43 +4115,81 @@ class Daemon(metaclass=JSONRPCServerType): return self.wallet_manager.get_transaction(txid) TXO_DOC = """ - List transaction outputs. + List and sum transaction outputs. """ + @staticmethod + def _constrain_txo_from_kwargs( + constraints, type=None, txid=None, # pylint: disable=redefined-builtin + claim_id=None, channel_id=None, name=None, unspent=False, reposted_claim_id=None, + is_my_input_or_output=None, exclude_internal_transfers=False, + is_my_output=None, is_not_my_output=None, + is_my_input=None, is_not_my_input=None): + constraints['unspent'] = unspent + constraints['exclude_internal_transfers'] = exclude_internal_transfers + if is_my_input_or_output is True: + constraints['is_my_input_or_output'] = True + else: + if is_my_input is True: + constraints['is_my_input'] = True + elif is_not_my_input is True: + constraints['is_my_input'] = False + if is_my_output is True: + constraints['is_my_output'] = True + elif is_not_my_output is True: + constraints['is_my_output'] = False + database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) + database.constrain_single_or_list(constraints, 'channel_id', channel_id) + database.constrain_single_or_list(constraints, 'claim_id', claim_id) + database.constrain_single_or_list(constraints, 'claim_name', name) + database.constrain_single_or_list(constraints, 'txid', txid) + database.constrain_single_or_list(constraints, 'reposted_claim_id', reposted_claim_id) + return constraints + @requires(WALLET_COMPONENT) def jsonrpc_txo_list( - self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin - claim_id=None, name=None, unspent=False, - include_is_received=False, is_received=None, is_not_received=None, - wallet_id=None, page=None, page_size=None, resolve=False): + self, account_id=None, wallet_id=None, page=None, page_size=None, + resolve=False, order_by=None, no_totals=False, **kwargs): """ List my transaction outputs. Usage: - txo_list [--account_id=] [--type=...] [--txid=...] - [--claim_id=...] [--name=...] [--unspent] - [--include_is_received] [--is_received] [--is_not_received] - [--wallet_id=] [--include_is_received] [--is_received] - [--page=] [--page_size=] - [--resolve] + txo_list [--account_id=] [--type=...] [--txid=...] [--unspent] + [--claim_id=...] [--channel_id=...] [--name=...] + [--is_my_input_or_output | + [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] + ] + [--exclude_internal_transfers] + [--wallet_id=] [--page=] [--page_size=] + [--resolve] [--order_by=][--no_totals] Options: --type= : (str or list) claim type: stream, channel, support, purchase, collection, repost, other --txid= : (str or list) transaction id of outputs - --unspent : (bool) hide spent outputs, show only unspent ones --claim_id= : (str or list) claim id + --channel_id= : (str or list) claims in this channel --name= : (str or list) claim name - --include_is_received : (bool) calculate the is_received property and - include in output, this happens automatically if you - use the --is_received or --is_not_received filters - --is_received : (bool) only return txos sent from others to this account - --is_not_received : (bool) only return txos created by this account + --unspent : (bool) hide spent outputs, show only unspent ones + --is_my_input_or_output : (bool) txos which have your inputs or your outputs, + if using this flag the other related flags + are ignored (--is_my_output, --is_my_input, etc) + --is_my_output : (bool) show outputs controlled by you + --is_not_my_output : (bool) show outputs not controlled by you + --is_my_input : (bool) show outputs created by you + --is_not_my_input : (bool) show outputs not created by you + --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: + "--is_my_input --is_my_output --type=other" + this allows to exclude "change" payments, this + flag can be used in combination with any of the other flags --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata + --order_by= : (str) field to order by: 'name', 'height', 'amount' and 'none' + --no_totals : (bool) do not calculate the total number of pages and items in result set + (significant performance boost) Returns: {Paginated[Output]} """ @@ -4155,15 +4201,63 @@ class Daemon(metaclass=JSONRPCServerType): else: claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts, read_only=True) claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts, read_only=True) - constraints = {'resolve': resolve, 'unspent': unspent, 'include_is_received': include_is_received} - if is_received is True: - constraints['is_received'] = True - elif is_not_received is True: - constraints['is_received'] = False - database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) - database.constrain_single_or_list(constraints, 'claim_id', claim_id) - database.constrain_single_or_list(constraints, 'claim_name', name) - return paginate_rows(claims, claim_count, page, page_size, **constraints) + constraints = { + 'resolve': resolve, + 'include_is_spent': True, + 'include_is_my_input': True, + 'include_is_my_output': True, + } + if order_by is not None: + if order_by == 'name': + constraints['order_by'] = 'txo.claim_name' + elif order_by in ('height', 'amount', 'none'): + constraints['order_by'] = order_by + else: + raise ValueError(f"'{order_by}' is not a valid --order_by value.") + self._constrain_txo_from_kwargs(constraints, **kwargs) + return paginate_rows(claims, None if no_totals else claim_count, page, page_size, **constraints) + + @requires(WALLET_COMPONENT) + def jsonrpc_txo_sum(self, account_id=None, wallet_id=None, **kwargs): + """ + Sum of transaction outputs. + + Usage: + txo_list [--account_id=] [--type=...] [--txid=...] + [--claim_id=...] [--name=...] [--unspent] + [--is_my_input_or_output | + [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] + ] + [--exclude_internal_transfers] [--wallet_id=] + + Options: + --type= : (str or list) claim type: stream, channel, support, + purchase, collection, repost, other + --txid= : (str or list) transaction id of outputs + --claim_id= : (str or list) claim id + --name= : (str or list) claim name + --unspent : (bool) hide spent outputs, show only unspent ones + --is_my_input_or_output : (bool) txos which have your inputs or your outputs, + if using this flag the other related flags + are ignored (--is_my_output, --is_my_input, etc) + --is_my_output : (bool) show outputs controlled by you + --is_not_my_output : (bool) show outputs not controlled by you + --is_my_input : (bool) show outputs created by you + --is_not_my_input : (bool) show outputs not created by you + --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: + "--is_my_input --is_my_output --type=other" + this allows to exclude "change" payments, this + flag can be used in combination with any of the other flags + --account_id= : (str) id of the account to query + --wallet_id= : (str) restrict results to specific wallet + + Returns: int + """ + wallet = self.wallet_manager.get_wallet_or_default(wallet_id) + return self.ledger.get_txo_sum( + wallet=wallet, accounts=[wallet.get_account_or_error(account_id)] if account_id else wallet.accounts, + read_only=True, **self._constrain_txo_from_kwargs({}, **kwargs) + ) UTXO_DOC = """ Unspent transaction management. diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index a3c9e318d..da87da92e 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -168,14 +168,14 @@ class JSONResponseEncoder(JSONEncoder): 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, 'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None } - if txo.is_change is not None: - output['is_change'] = txo.is_change - if txo.is_received is not None: - output['is_received'] = txo.is_received if txo.is_spent is not None: output['is_spent'] = txo.is_spent - if txo.is_my_account is not None: - output['is_mine'] = txo.is_my_account + if txo.is_my_output is not None: + output['is_my_output'] = txo.is_my_output + if txo.is_my_input is not None: + output['is_my_input'] = txo.is_my_input + if txo.is_internal_transfer is not None: + output['is_internal_transfer'] = txo.is_internal_transfer if txo.script.is_claim_name: output['type'] = 'claim' diff --git a/lbry/testcase.py b/lbry/testcase.py index 418adf2a2..798937db6 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -555,6 +555,11 @@ class CommandTestCase(IntegrationTestCase): self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm ) + async def wallet_send(self, *args, confirm=True, **kwargs): + return await self.confirm_and_render( + self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm + ) + async def resolve(self, uri): return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri] @@ -567,6 +572,9 @@ class CommandTestCase(IntegrationTestCase): async def txo_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items'] + async def txo_sum(self, *args, **kwargs): + return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs)) + async def claim_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items'] diff --git a/lbry/wallet/account.py b/lbry/wallet/account.py index b7d4e29c5..c37457c5d 100644 --- a/lbry/wallet/account.py +++ b/lbry/wallet/account.py @@ -437,7 +437,7 @@ class Account: async def get_addresses(self, read_only=False, **constraints) -> List[str]: rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints) - return [r[0] for r in rows] + return [r['address'] for r in rows] def get_address_records(self, **constraints): return self.ledger.db.get_addresses(accounts=[self], **constraints) @@ -454,7 +454,7 @@ class Account: def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints): if not include_claims: - constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])}) + constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])}) if confirmations > 0: height = self.ledger.headers.height - (confirmations-1) constraints.update({'height__lte': height, 'height__gt': 0}) @@ -569,14 +569,13 @@ class Account: total = await get_total_balance() if reserved_subtotals: claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES) - for amount, spent, from_me, to_me, height in await self.get_support_summary(): - if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1): + for txo in await self.get_support_summary(): + if confirmations > 0 and not 0 < txo.tx_ref.height <= self.ledger.headers.height - (confirmations - 1): continue - if not spent and to_me: - if from_me: - supports_balance += amount - else: - tips_balance += amount + if txo.is_my_input: + supports_balance += txo.amount + else: + tips_balance += txo.amount reserved = claims_balance + supports_balance + tips_balance else: reserved = await self.get_balance( @@ -634,7 +633,7 @@ class Account: return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints) def get_support_summary(self): - return self.ledger.db.get_supports_summary(account_id=self.id) + return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self]) async def release_all_outputs(self): await self.ledger.db.release_all_outputs(self) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 4c733fa14..d8265459e 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -29,6 +29,7 @@ reader_context: Optional[ContextVar[ReaderProcessState]] = ContextVar('reader_co def initializer(path): db = sqlite3.connect(path) + db.row_factory = dict_row_factory db.executescript("pragma journal_mode=WAL;") reader = ReaderProcessState(db.cursor()) reader_context.set(reader) @@ -106,7 +107,7 @@ class AIOSQLite: return self.run(lambda conn: conn.executescript(script)) async def _execute_fetch(self, sql: str, parameters: Iterable = None, - read_only=False, fetch_all: bool = False) -> Iterable[sqlite3.Row]: + read_only=False, fetch_all: bool = False) -> List[dict]: read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone parameters = parameters if parameters is not None else [] if read_only: @@ -120,11 +121,11 @@ class AIOSQLite: return await self.run(lambda conn: conn.execute(sql, parameters).fetchone()) async def execute_fetchall(self, sql: str, parameters: Iterable = None, - read_only=False) -> Iterable[sqlite3.Row]: + read_only=False) -> List[dict]: return await self._execute_fetch(sql, parameters, read_only, fetch_all=True) async def execute_fetchone(self, sql: str, parameters: Iterable = None, - read_only=False) -> Iterable[sqlite3.Row]: + read_only=False) -> List[dict]: return await self._execute_fetch(sql, parameters, read_only, fetch_all=False) def execute(self, sql: str, parameters: Iterable = None) -> Awaitable[sqlite3.Cursor]: @@ -294,13 +295,6 @@ def interpolate(sql, values): return sql -def rows_to_dict(rows, fields): - if rows: - return [dict(zip(fields, r)) for r in rows] - else: - return [] - - def constrain_single_or_list(constraints, column, value, convert=lambda x: x): if value is not None: if isinstance(value, list): @@ -384,9 +378,16 @@ class SQLiteMixin: return sql, values +def dict_row_factory(cursor, row): + d = {} + for idx, col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + + class Database(SQLiteMixin): - SCHEMA_VERSION = "1.1" + SCHEMA_VERSION = "1.2" PRAGMAS = """ pragma journal_mode=WAL; @@ -438,23 +439,29 @@ class Database(SQLiteMixin): txo_type integer not null default 0, claim_id text, - claim_name text + claim_name text, + + channel_id text, + reposted_claim_id text ); create index if not exists txo_txid_idx on txo (txid); create index if not exists txo_address_idx on txo (address); create index if not exists txo_claim_id_idx on txo (claim_id); create index if not exists txo_claim_name_idx on txo (claim_name); create index if not exists txo_txo_type_idx on txo (txo_type); + create index if not exists txo_channel_id_idx on txo (channel_id); + create index if not exists txo_reposted_claim_idx on txo (reposted_claim_id); """ CREATE_TXI_TABLE = """ create table if not exists txi ( txid text references tx, - txoid text references txo, - address text references pubkey_address + txoid text references txo primary key, + address text references pubkey_address, + position integer not null ); create index if not exists txi_address_idx on txi (address); - create index if not exists txi_txoid_idx on txi (txoid); + create index if not exists first_input_idx on txi (txid, address) where position=0; """ CREATE_TABLES_QUERY = ( @@ -466,19 +473,27 @@ class Database(SQLiteMixin): CREATE_TXI_TABLE ) - @staticmethod - def txo_to_row(tx, address, txo): + async def open(self): + await super().open() + self.db.writer_connection.row_factory = dict_row_factory + + def txo_to_row(self, tx, txo): row = { 'txid': tx.id, 'txoid': txo.id, - 'address': address, + 'address': txo.get_address(self.ledger), 'position': txo.position, 'amount': txo.amount, 'script': sqlite3.Binary(txo.script.source) } if txo.is_claim: if txo.can_decode_claim: - row['txo_type'] = TXO_TYPES.get(txo.claim.claim_type, TXO_TYPES['stream']) + claim = txo.claim + row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream']) + if claim.is_repost: + row['reposted_claim_id'] = claim.repost.reference.claim_id + if claim.is_signed: + row['channel_id'] = claim.signing_channel_id else: row['txo_type'] = TXO_TYPES['stream'] elif txo.is_support: @@ -517,25 +532,29 @@ class Database(SQLiteMixin): def _transaction_io(self, conn: sqlite3.Connection, tx: Transaction, address, txhash): conn.execute(*self._insert_sql('tx', self.tx_to_row(tx), replace=True)).fetchall() - for txo in tx.outputs: - if txo.script.is_pay_pubkey_hash and txo.pubkey_hash == txhash: - conn.execute(*self._insert_sql( - "txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True - )).fetchall() - elif txo.script.is_pay_script_hash: - # TODO: implement script hash payments - log.warning('Database.save_transaction_io: pay script hash is not implemented!') + is_my_input = False for txi in tx.inputs: if txi.txo_ref.txo is not None: txo = txi.txo_ref.txo if txo.has_address and txo.get_address(self.ledger) == address: + is_my_input = True conn.execute(*self._insert_sql("txi", { 'txid': tx.id, 'txoid': txo.id, 'address': address, + 'position': txi.position }, ignore_duplicate=True)).fetchall() + for txo in tx.outputs: + if txo.script.is_pay_pubkey_hash and (txo.pubkey_hash == txhash or is_my_input): + conn.execute(*self._insert_sql( + "txo", self.txo_to_row(tx, txo), ignore_duplicate=True + )).fetchall() + elif txo.script.is_pay_script_hash: + # TODO: implement script hash payments + log.warning('Database.save_transaction_io: pay script hash is not implemented!') + def save_transaction_io(self, tx: Transaction, address, txhash, history): return self.save_transaction_io_batch([tx], address, txhash, history) @@ -581,9 +600,13 @@ class Database(SQLiteMixin): *query(f"SELECT {cols} FROM tx", **constraints), read_only=read_only ) - TXO_NOT_MINE = Output(None, None, is_my_account=False) + TXO_NOT_MINE = Output(None, None, is_my_output=False) async def get_transactions(self, wallet=None, **constraints): + include_is_spent = constraints.pop('include_is_spent', False) + include_is_my_input = constraints.pop('include_is_my_input', False) + include_is_my_output = constraints.pop('include_is_my_output', False) + tx_rows = await self.select_transactions( 'txid, raw, height, position, is_verified', order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]), @@ -595,9 +618,10 @@ class Database(SQLiteMixin): txids, txs, txi_txoids = [], [], [] for row in tx_rows: - txids.append(row[0]) + txids.append(row['txid']) txs.append(Transaction( - raw=row[1], height=row[2], position=row[3], is_verified=bool(row[4]) + raw=row['raw'], height=row['height'], position=row['position'], + is_verified=bool(row['is_verified']) )) for txi in txs[-1].inputs: txi_txoids.append(txi.txo_ref.id) @@ -609,7 +633,10 @@ class Database(SQLiteMixin): txo.id: txo for txo in (await self.get_txos( wallet=wallet, - txid__in=txids[offset:offset+step], + txid__in=txids[offset:offset+step], order_by='txo.txid', + include_is_spent=include_is_spent, + include_is_my_input=include_is_my_input, + include_is_my_output=include_is_my_output, )) }) @@ -619,7 +646,8 @@ class Database(SQLiteMixin): txo.id: txo for txo in (await self.get_txos( wallet=wallet, - txoid__in=txi_txoids[offset:offset+step], + txoid__in=txi_txoids[offset:offset+step], order_by='txo.txoid', + include_is_my_output=include_is_my_output, )) }) @@ -647,82 +675,151 @@ class Database(SQLiteMixin): constraints.pop('offset', None) constraints.pop('limit', None) constraints.pop('order_by', None) - count = await self.select_transactions('count(*)', **constraints) - return count[0][0] + count = await self.select_transactions('COUNT(*) as total', **constraints) + return count[0]['total'] async def get_transaction(self, **constraints): txs = await self.get_transactions(limit=1, **constraints) if txs: return txs[0] - async def select_txos(self, cols, wallet=None, include_is_received=False, read_only=False, **constraints): - if include_is_received: - assert wallet is not None, 'cannot use is_recieved filter without wallet argument' - account_in_wallet, values = constraints_to_sql({ - '$$account__in#is_received': [a.public_key.address for a in wallet.accounts] + async def select_txos( + self, cols, accounts=None, is_my_input=None, is_my_output=True, + is_my_input_or_output=None, exclude_internal_transfers=False, + include_is_spent=False, include_is_my_input=False, + read_only=False, **constraints): + for rename_col in ('txid', 'txoid'): + for rename_constraint in (rename_col, rename_col+'__in', rename_col+'__not_in'): + if rename_constraint in constraints: + constraints['txo.'+rename_constraint] = constraints.pop(rename_constraint) + if accounts: + account_in_sql, values = constraints_to_sql({ + '$$account__in': [a.public_key.address for a in accounts] }) - cols += f""", - NOT EXISTS( - SELECT 1 FROM txi JOIN account_address USING (address) - WHERE txi.txid=txo.txid AND {account_in_wallet} - ) as is_received - """ + my_addresses = f"SELECT address FROM account_address WHERE {account_in_sql}" constraints.update(values) - sql = f"SELECT {cols} FROM txo JOIN tx USING (txid)" - if 'accounts' in constraints: - sql += " JOIN account_address USING (address)" - return await self.db.execute_fetchall(*query(sql, **constraints), read_only=read_only) + if is_my_input_or_output: + include_is_my_input = True + constraints['received_or_sent__or'] = { + 'txo.address__in': my_addresses, + 'sent__and': { + 'txi.address__is_not_null': True, + 'txi.address__in': my_addresses + } + } + else: + if is_my_output: + constraints['txo.address__in'] = my_addresses + elif is_my_output is False: + constraints['txo.address__not_in'] = my_addresses + if is_my_input: + include_is_my_input = True + constraints['txi.address__is_not_null'] = True + constraints['txi.address__in'] = my_addresses + elif is_my_input is False: + include_is_my_input = True + constraints['is_my_input_false__or'] = { + 'txi.address__is_null': True, + 'txi.address__not_in': my_addresses + } + if exclude_internal_transfers: + include_is_my_input = True + constraints['exclude_internal_payments__or'] = { + 'txo.txo_type__not': TXO_TYPES['other'], + 'txi.address__is_null': True, + 'txi.address__not_in': my_addresses + } + sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"] + if include_is_spent: + sql.append("LEFT JOIN txi AS spent ON (spent.txoid=txo.txoid)") + if include_is_my_input: + sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)") + return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only) @staticmethod def constrain_unspent(constraints): constraints['is_reserved'] = False - constraints['txoid__not_in'] = "SELECT txoid FROM txi" + constraints['include_is_spent'] = True + constraints['spent.txoid__is_null'] = True + + async def get_txos(self, wallet=None, no_tx=False, unspent=False, read_only=False, **constraints): - async def get_txos(self, wallet=None, no_tx=False, unspent=False, include_is_received=False, - read_only=False, **constraints): - include_is_received = include_is_received or 'is_received' in constraints if unspent: self.constrain_unspent(constraints) + + include_is_spent = constraints.get('include_is_spent', False) + include_is_my_input = constraints.get('include_is_my_input', False) + include_is_my_output = constraints.pop('include_is_my_output', False) + + select_columns = [ + "tx.txid, raw, tx.height, tx.position as tx_position, tx.is_verified, " + "txo_type, txo.position as txo_position, amount, script" + ] + my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set() - if 'order_by' not in constraints: + my_accounts_sql = "" + if include_is_my_output or include_is_my_input: + my_accounts_sql, values = constraints_to_sql({'$$account__in#_wallet': my_accounts}) + constraints.update(values) + + if include_is_my_output and my_accounts: + if constraints.get('is_my_output', None) in (True, False): + select_columns.append(f"{1 if constraints['is_my_output'] else 0} AS is_my_output") + else: + select_columns.append(f"""( + txo.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) + ) AS is_my_output""") + + if include_is_my_input and my_accounts: + if constraints.get('is_my_input', None) in (True, False): + select_columns.append(f"{1 if constraints['is_my_input'] else 0} AS is_my_input") + else: + select_columns.append(f"""( + txi.address IS NOT NULL AND + txi.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) + ) AS is_my_input + """) + + if include_is_spent: + select_columns.append("spent.txoid IS NOT NULL AS is_spent") + + if 'order_by' not in constraints or constraints['order_by'] == 'height': constraints['order_by'] = [ "tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position" ] - rows = await self.select_txos( - """ - tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, ( - select group_concat(account||"|"||chain) from account_address - where account_address.address=txo.address - ), exists(select 1 from txi where txi.txoid=txo.txoid) - """, - wallet=wallet, include_is_received=include_is_received, read_only=read_only, **constraints - ) + elif constraints.get('order_by', None) == 'none': + del constraints['order_by'] + + rows = await self.select_txos(', '.join(select_columns), read_only=read_only, **constraints) + txos = [] txs = {} for row in rows: if no_tx: txo = Output( - amount=row[6], - script=OutputScript(row[7]), - tx_ref=TXRefImmutable.from_id(row[0], row[2]), - position=row[5] + amount=row['amount'], + script=OutputScript(row['script']), + tx_ref=TXRefImmutable.from_id(row['txid'], row['height']), + position=row['txo_position'] ) else: - if row[0] not in txs: - txs[row[0]] = Transaction( - row[1], height=row[2], position=row[3], is_verified=row[4] + if row['txid'] not in txs: + txs[row['txid']] = Transaction( + row['raw'], height=row['height'], position=row['tx_position'], + is_verified=bool(row['is_verified']) ) - txo = txs[row[0]].outputs[row[5]] - row_accounts = dict(a.split('|') for a in row[8].split(',')) - account_match = set(row_accounts) & my_accounts - txo.is_spent = bool(row[9]) - if include_is_received: - txo.is_received = bool(row[10]) - if account_match: - txo.is_my_account = True - txo.is_change = row_accounts[account_match.pop()] == '1' - else: - txo.is_change = txo.is_my_account = False + txo = txs[row['txid']].outputs[row['txo_position']] + if include_is_spent: + txo.is_spent = bool(row['is_spent']) + if include_is_my_input: + txo.is_my_input = bool(row['is_my_input']) + if include_is_my_output: + txo.is_my_output = bool(row['is_my_output']) + if include_is_my_input and include_is_my_output: + if txo.is_my_input and txo.is_my_output and row['txo_type'] == TXO_TYPES['other']: + txo.is_internal_transfer = True + else: + txo.is_internal_transfer = False txos.append(txo) channel_ids = set() @@ -754,16 +851,26 @@ class Database(SQLiteMixin): return txos - async def get_txo_count(self, unspent=False, **constraints): - constraints['include_is_received'] = 'is_received' in constraints + def _clean_txo_constraints_for_aggregation(self, unspent, constraints): + constraints.pop('include_is_my_input', None) + constraints.pop('include_is_my_output', None) + constraints.pop('wallet', None) constraints.pop('resolve', None) constraints.pop('offset', None) constraints.pop('limit', None) constraints.pop('order_by', None) if unspent: self.constrain_unspent(constraints) - count = await self.select_txos('count(*)', **constraints) - return count[0][0] + + async def get_txo_count(self, unspent=False, **constraints): + self._clean_txo_constraints_for_aggregation(unspent, constraints) + count = await self.select_txos('COUNT(*) as total', **constraints) + return count[0]['total'] + + async def get_txo_sum(self, unspent=False, **constraints): + self._clean_txo_constraints_for_aggregation(unspent, constraints) + result = await self.select_txos('SUM(amount) as total', **constraints) + return result[0]['total'] def get_utxos(self, read_only=False, **constraints): return self.get_txos(unspent=True, read_only=read_only, **constraints) @@ -776,8 +883,8 @@ class Database(SQLiteMixin): "'wallet' or 'accounts' constraints required to calculate balance" constraints['accounts'] = accounts or wallet.accounts self.constrain_unspent(constraints) - balance = await self.select_txos('SUM(amount)', read_only=read_only, **constraints) - return balance[0][0] or 0 + balance = await self.select_txos('SUM(amount) as total', read_only=read_only, **constraints) + return balance[0]['total'] or 0 async def select_addresses(self, cols, read_only=False, **constraints): return await self.db.execute_fetchall(*query( @@ -790,7 +897,7 @@ class Database(SQLiteMixin): 'address', 'account', 'chain', 'history', 'used_times', 'pubkey', 'chain_code', 'n', 'depth' ) - addresses = rows_to_dict(await self.select_addresses(', '.join(cols), read_only=read_only, **constraints), cols) + addresses = await self.select_addresses(', '.join(cols), read_only=read_only, **constraints) if 'pubkey' in cols: for address in addresses: address['pubkey'] = PubKey( @@ -800,8 +907,8 @@ class Database(SQLiteMixin): return addresses async def get_address_count(self, cols=None, read_only=False, **constraints): - count = await self.select_addresses('count(*)', read_only=read_only, **constraints) - return count[0][0] + count = await self.select_addresses('COUNT(*) as total', read_only=read_only, **constraints) + return count[0]['total'] async def get_address(self, read_only=False, **constraints): addresses = await self.get_addresses(read_only=read_only, limit=1, **constraints) @@ -932,13 +1039,11 @@ class Database(SQLiteMixin): " )", (account.public_key.address, ) ) - def get_supports_summary(self, account_id, read_only=False): - return self.db.execute_fetchall(f""" - select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent, - (txo.txid in - (select txi.txid from txi join account_address a on txi.address = a.address - where a.account = ?)) as from_me, - (txo.address in (select address from account_address where account=?)) as to_me, - tx.height - from txo join tx using (txid) where txo_type={TXO_TYPES['support']} - """, (account_id, account_id), read_only=read_only) + def get_supports_summary(self, read_only=False, **constraints): + return self.get_txos( + txo_type=TXO_TYPES['support'], + unspent=True, is_my_output=True, + include_is_my_input=True, + no_tx=True, read_only=read_only, + **constraints + ) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 64ac744c0..639446d3f 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -274,6 +274,9 @@ class Ledger(metaclass=LedgerRegistry): def get_txo_count(self, **constraints): return self.db.get_txo_count(**constraints) + def get_txo_sum(self, **constraints): + return self.db.get_txo_sum(**constraints) + def get_transactions(self, **constraints): return self.db.get_transactions(**constraints) @@ -521,7 +524,9 @@ class Ledger(metaclass=LedgerRegistry): check_db_for_txos.append(txi.txo_ref.id) referenced_txos = {} if not check_db_for_txos else { - txo.id: txo for txo in await self.db.get_txos(txoid__in=check_db_for_txos, no_tx=True) + txo.id: txo for txo in await self.db.get_txos( + txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True + ) } for txi in tx.inputs: @@ -826,7 +831,10 @@ class Ledger(metaclass=LedgerRegistry): return self.db.get_support_count(**constraints) async def get_transaction_history(self, read_only=False, **constraints): - txs: List[Transaction] = await self.db.get_transactions(read_only=read_only, **constraints) + txs: List[Transaction] = await self.db.get_transactions( + include_is_my_output=True, include_is_spent=True, + read_only=read_only, **constraints + ) headers = self.headers history = [] for tx in txs: # pylint: disable=too-many-nested-blocks @@ -842,7 +850,7 @@ class Ledger(metaclass=LedgerRegistry): 'abandon_info': [], 'purchase_info': [] } - is_my_inputs = all([txi.is_my_account for txi in tx.inputs]) + is_my_inputs = all([txi.is_my_input for txi in tx.inputs]) if is_my_inputs: # fees only matter if we are the ones paying them item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee) diff --git a/lbry/wallet/transaction.py b/lbry/wallet/transaction.py index cfd637687..264357f56 100644 --- a/lbry/wallet/transaction.py +++ b/lbry/wallet/transaction.py @@ -159,11 +159,11 @@ class Input(InputOutput): return self.txo_ref.txo.amount @property - def is_my_account(self) -> Optional[bool]: + def is_my_input(self) -> Optional[bool]: """ True if the output this input spends is yours. """ if self.txo_ref.txo is None: return False - return self.txo_ref.txo.is_my_account + return self.txo_ref.txo.is_my_output @classmethod def deserialize_from(cls, stream): @@ -207,7 +207,7 @@ class OutputEffectiveAmountEstimator: class Output(InputOutput): __slots__ = ( - 'amount', 'script', 'is_change', 'is_spent', 'is_received', 'is_my_account', + 'amount', 'script', 'is_internal_transfer', 'is_spent', 'is_my_output', 'is_my_input', 'channel', 'private_key', 'meta', 'purchase', 'purchased_claim', 'purchase_receipt', 'reposted_claim', 'claims', @@ -215,17 +215,17 @@ class Output(InputOutput): def __init__(self, amount: int, script: OutputScript, tx_ref: TXRef = None, position: int = None, - is_change: Optional[bool] = None, is_spent: Optional[bool] = None, - is_received: Optional[bool] = None, is_my_account: Optional[bool] = None, + is_internal_transfer: Optional[bool] = None, is_spent: Optional[bool] = None, + is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None, channel: Optional['Output'] = None, private_key: Optional[str] = None ) -> None: super().__init__(tx_ref, position) self.amount = amount self.script = script - self.is_change = is_change + self.is_internal_transfer = is_internal_transfer self.is_spent = is_spent - self.is_received = is_received - self.is_my_account = is_my_account + self.is_my_output = is_my_output + self.is_my_input = is_my_input self.channel = channel self.private_key = private_key self.purchase: 'Output' = None # txo containing purchase metadata @@ -235,15 +235,17 @@ class Output(InputOutput): self.claims: List['Output'] = None # resolved claims for collection self.meta = {} - def update_annotations(self, annotated): + def update_annotations(self, annotated: 'Output'): if annotated is None: - self.is_change = None + self.is_internal_transfer = None self.is_spent = None - self.is_my_account = None + self.is_my_output = None + self.is_my_input = None else: - self.is_change = annotated.is_change + self.is_internal_transfer = annotated.is_internal_transfer self.is_spent = annotated.is_spent - self.is_my_account = annotated.is_my_account + self.is_my_output = annotated.is_my_output + self.is_my_input = annotated.is_my_input self.channel = annotated.channel if annotated else None self.private_key = annotated.private_key if annotated else None @@ -592,21 +594,21 @@ class Transaction: for txi in self.inputs: if txi.txo_ref.txo is None: continue - if txi.is_my_account is None: - raise ValueError( - "Cannot access net_account_balance if inputs/outputs do not " - "have is_my_account set properly." - ) - if txi.is_my_account: + if txi.is_my_input is True: balance -= txi.amount - for txo in self.outputs: - if txo.is_my_account is None: + elif txi.is_my_input is None: raise ValueError( - "Cannot access net_account_balance if inputs/outputs do not " - "have is_my_account set properly." + "Cannot access net_account_balance if inputs do not " + "have is_my_input set properly." ) - if txo.is_my_account: + for txo in self.outputs: + if txo.is_my_output is True: balance += txo.amount + elif txo.is_my_output is None: + raise ValueError( + "Cannot access net_account_balance if outputs do not " + "have is_my_output set properly." + ) return balance @property @@ -751,7 +753,7 @@ class Transaction: change_hash160 = change_account.ledger.address_to_hash160(change_address) change_amount = change - cost_of_change change_output = Output.pay_pubkey_hash(change_amount, change_hash160) - change_output.is_change = True + change_output.is_internal_transfer = True tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)]) if tx._outputs: @@ -856,17 +858,17 @@ class Transaction: @property def my_inputs(self): for txi in self.inputs: - if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_account: + if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_output: yield txi def _filter_my_outputs(self, f): for txo in self.outputs: - if txo.is_my_account and f(txo.script): + if txo.is_my_output and f(txo.script): yield txo def _filter_other_outputs(self, f): for txo in self.outputs: - if not txo.is_my_account and f(txo.script): + if not txo.is_my_output and f(txo.script): yield txo def _filter_any_outputs(self, f): @@ -898,7 +900,7 @@ class Transaction: def my_abandon_outputs(self): for txi in self.inputs: abandon = txi.txo_ref.txo - if abandon is not None and abandon.is_my_account and abandon.script.is_claim_involved: + if abandon is not None and abandon.is_my_output and abandon.script.is_claim_involved: is_update = False if abandon.script.is_claim_name or abandon.script.is_update_claim: for update in self.my_update_outputs: diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index a99106751..7e3729cbc 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -9,6 +9,7 @@ from lbry.error import InsufficientFundsError from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE from lbry.testcase import CommandTestCase from lbry.wallet.transaction import Transaction +from lbry.wallet.util import satoshis_to_coins as lbc log = logging.getLogger(__name__) @@ -420,11 +421,21 @@ class TransactionCommands(ClaimTestCase): class TransactionOutputCommands(ClaimTestCase): - async def test_txo_list_filtering(self): + async def test_txo_list_and_sum_filtering(self): channel_id = self.get_claim_id(await self.channel_create()) + self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', unspent=True))) await self.channel_update(channel_id, bid='0.5') - stream_id = self.get_claim_id(await self.stream_create()) - await self.stream_update(stream_id, bid='0.5') + self.assertEqual('0.5', lbc(await self.txo_sum(type='channel', unspent=True))) + self.assertEqual('1.5', lbc(await self.txo_sum(type='channel'))) + + stream_id = self.get_claim_id(await self.stream_create(bid='1.3')) + self.assertEqual('1.3', lbc(await self.txo_sum(type='stream', unspent=True))) + await self.stream_update(stream_id, bid='0.7') + self.assertEqual('0.7', lbc(await self.txo_sum(type='stream', unspent=True))) + self.assertEqual('2.0', lbc(await self.txo_sum(type='stream'))) + + self.assertEqual('1.2', lbc(await self.txo_sum(type=['stream', 'channel'], unspent=True))) + self.assertEqual('3.5', lbc(await self.txo_sum(type=['stream', 'channel']))) # type filtering r = await self.txo_list(type='channel') @@ -480,25 +491,69 @@ class TransactionOutputCommands(ClaimTestCase): self.assertTrue(r[0]['is_spent']) self.assertTrue(r[1]['is_spent']) - async def test_txo_list_received_filtering(self): + async def test_txo_list_my_input_output_filtering(self): wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True) address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id) - await self.channel_create(claim_address=address2) + await self.channel_create('@kept-channel') + await self.channel_create('@sent-channel', claim_address=address2) - r = await self.txo_list(include_is_received=True) - self.assertEqual(2, len(r)) - self.assertFalse(r[0]['is_received']) - self.assertTrue(r[1]['is_received']) - rt = await self.txo_list(is_not_received=True) - self.assertEqual(1, len(rt)) - self.assertEqual(rt[0], r[0]) - rf = await self.txo_list(is_received=True) - self.assertEqual(1, len(rf)) - self.assertEqual(rf[0], r[1]) + # all txos on second wallet + received_channel, = await self.txo_list(wallet_id=wallet2.id, is_my_input_or_output=True) + self.assertEqual('1.0', received_channel['amount']) + self.assertFalse(received_channel['is_my_input']) + self.assertTrue(received_channel['is_my_output']) + self.assertFalse(received_channel['is_internal_transfer']) - r = await self.txo_list(include_is_received=True, wallet_id=wallet2.id) - self.assertEqual(1, len(r)) - self.assertTrue(r[0]['is_received']) + # all txos on default wallet + r = await self.txo_list(is_my_input_or_output=True) + self.assertEqual( + ['1.0', '7.947786', '1.0', '8.973893', '10.0'], + [t['amount'] for t in r] + ) + + sent_channel, change2, kept_channel, change1, initial_funds = r + + self.assertTrue(sent_channel['is_my_input']) + self.assertFalse(sent_channel['is_my_output']) + self.assertFalse(sent_channel['is_internal_transfer']) + self.assertTrue(change2['is_my_input']) + self.assertTrue(change2['is_my_output']) + self.assertTrue(change2['is_internal_transfer']) + + self.assertTrue(kept_channel['is_my_input']) + self.assertTrue(kept_channel['is_my_output']) + self.assertFalse(kept_channel['is_internal_transfer']) + self.assertTrue(change1['is_my_input']) + self.assertTrue(change1['is_my_output']) + self.assertTrue(change1['is_internal_transfer']) + + self.assertFalse(initial_funds['is_my_input']) + self.assertTrue(initial_funds['is_my_output']) + self.assertFalse(initial_funds['is_internal_transfer']) + + # my stuff and stuff i sent excluding "change" + r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True) + self.assertEqual([sent_channel, kept_channel, initial_funds], r) + + # my unspent stuff and stuff i sent excluding "change" + r = await self.txo_list(is_my_input_or_output=True, unspent=True, exclude_internal_transfers=True) + self.assertEqual([sent_channel, kept_channel], r) + + # only "change" + r = await self.txo_list(is_my_input=True, is_my_output=True, type="other") + self.assertEqual([change2, change1], r) + + # only unspent "change" + r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", unspent=True) + self.assertEqual([change2], r) + + # all my unspent stuff + r = await self.txo_list(is_my_output=True, unspent=True) + self.assertEqual([change2, kept_channel], r) + + # stuff i sent + r = await self.txo_list(is_not_my_output=True) + self.assertEqual([sent_channel], r) class ClaimCommands(ClaimTestCase): @@ -617,7 +672,23 @@ class ClaimCommands(ClaimTestCase): self.assertTrue(r[1]['meta']['is_controlling']) # check that metadata is transfered - self.assertTrue(r[0]['is_mine']) + self.assertTrue(r[0]['is_my_output']) + + async def assertClaimList(self, claim_ids, **kwargs): + self.assertEqual(claim_ids, [c['claim_id'] for c in await self.claim_list(**kwargs)]) + + async def test_list_streams_in_channel_and_order_by(self): + channel1_id = self.get_claim_id(await self.channel_create('@chan-one')) + channel2_id = self.get_claim_id(await self.channel_create('@chan-two')) + stream1_id = self.get_claim_id(await self.stream_create('stream-a', bid='0.3', channel_id=channel1_id)) + stream2_id = self.get_claim_id(await self.stream_create('stream-b', bid='0.9', channel_id=channel1_id)) + stream3_id = self.get_claim_id(await self.stream_create('stream-c', bid='0.6', channel_id=channel2_id)) + await self.assertClaimList([stream2_id, stream1_id], channel_id=channel1_id) + await self.assertClaimList([stream3_id], channel_id=channel2_id) + await self.assertClaimList([stream3_id, stream2_id, stream1_id], channel_id=[channel1_id, channel2_id]) + await self.assertClaimList([stream1_id, stream2_id, stream3_id], claim_type='stream', order_by='name') + await self.assertClaimList([stream1_id, stream3_id, stream2_id], claim_type='stream', order_by='amount') + await self.assertClaimList([stream3_id, stream2_id, stream1_id], claim_type='stream', order_by='height') class ChannelCommands(CommandTestCase): @@ -968,12 +1039,16 @@ class StreamCommands(ClaimTestCase): claim_id = self.get_claim_id(tx) self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0) + self.assertItemCount(await self.daemon.jsonrpc_txo_list(reposted_claim_id=claim_id), 0) + self.assertItemCount(await self.daemon.jsonrpc_txo_list(type='repost'), 0) tx = await self.stream_repost(claim_id, 'newstuff-again', '1.1') repost_id = self.get_claim_id(tx) self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 1) self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1) self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id) + self.assertEqual((await self.txo_list(reposted_claim_id=claim_id))[0]['claim_id'], repost_id) + self.assertEqual((await self.txo_list(type='repost'))[0]['claim_id'], repost_id) # tags are inherited (non-common / indexed tags) self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2) diff --git a/tests/integration/blockchain/test_transactions.py b/tests/integration/blockchain/test_transactions.py index 82aa9c16a..6a0ace201 100644 --- a/tests/integration/blockchain/test_transactions.py +++ b/tests/integration/blockchain/test_transactions.py @@ -122,13 +122,13 @@ class BasicTransactionTests(IntegrationTestCase): await self.blockchain.generate(1) await self.ledger.wait(tx) # confirmed - tx = (await account1.get_transactions())[1] + tx = (await account1.get_transactions(include_is_my_input=True, include_is_my_output=True))[1] self.assertEqual(satoshis_to_coins(tx.inputs[0].amount), '1.1') self.assertEqual(satoshis_to_coins(tx.inputs[1].amount), '1.1') self.assertEqual(satoshis_to_coins(tx.outputs[0].amount), '2.0') self.assertEqual(tx.outputs[0].get_address(self.ledger), address2) - self.assertFalse(tx.outputs[0].is_change) - self.assertTrue(tx.outputs[1].is_change) + self.assertTrue(tx.outputs[0].is_internal_transfer) + self.assertTrue(tx.outputs[1].is_internal_transfer) async def test_history_edge_cases(self): await self.blockchain.generate(300) diff --git a/tests/integration/blockchain/test_wallet_commands.py b/tests/integration/blockchain/test_wallet_commands.py index 8500efa3c..c65fdfb10 100644 --- a/tests/integration/blockchain/test_wallet_commands.py +++ b/tests/integration/blockchain/test_wallet_commands.py @@ -82,6 +82,11 @@ class WalletCommands(CommandTestCase): async def test_granular_balances(self): account2 = await self.daemon.jsonrpc_account_create("Tip-er") + wallet2 = await self.daemon.jsonrpc_wallet_create('foo', create_account=True) + account3 = wallet2.default_account + address3 = await self.daemon.jsonrpc_address_unused(account3.id, wallet2.id) + await self.confirm_tx(await self.blockchain.send_to_address(address3, 1)) + await self.generate(1) account_balance = self.daemon.jsonrpc_account_balance wallet_balance = self.daemon.jsonrpc_wallet_balance @@ -128,7 +133,7 @@ class WalletCommands(CommandTestCase): # tip received support1 = await self.support_create( - self.get_claim_id(stream1), '0.3', tip=True, funding_account_ids=[account2.id] + self.get_claim_id(stream1), '0.3', tip=True, wallet_id=wallet2.id ) self.assertEqual(await account_balance(), { 'total': '9.27741', @@ -137,8 +142,8 @@ class WalletCommands(CommandTestCase): 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'} }) self.assertEqual(await wallet_balance(), { - 'total': '9.977268', - 'available': '6.677268', + 'total': '10.27741', + 'available': '6.97741', 'reserved': '3.3', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'} }) @@ -153,8 +158,8 @@ class WalletCommands(CommandTestCase): 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) self.assertEqual(await wallet_balance(), { - 'total': '9.977161', - 'available': '6.977161', + 'total': '10.277303', + 'available': '7.277303', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) @@ -165,17 +170,17 @@ class WalletCommands(CommandTestCase): # tip another claim await self.support_create( - self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id] + self.get_claim_id(stream2), '0.2', tip=True, wallet_id=wallet2.id ) self.assertEqual(await account_balance(), { - 'total': '9.077157', - 'available': '6.077157', + 'total': '9.277303', + 'available': '6.277303', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) self.assertEqual(await wallet_balance(), { - 'total': '9.938908', - 'available': '6.638908', + 'total': '10.439196', + 'available': '7.139196', 'reserved': '3.3', 'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'} }) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 64ad1a3f0..fab143d8c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -218,7 +218,7 @@ class DaemonDocsTests(TestCase): try: docopt.docopt(fn.__doc__, ()) except docopt.DocoptLanguageError as err: - failures.append(f"invalid docstring for {name}, {err.message}") + failures.append(f"invalid docstring for {name}, {err.args[0]}") except docopt.DocoptExit: pass if failures: diff --git a/tests/unit/wallet/test_database.py b/tests/unit/wallet/test_database.py index 50a40ad91..7ceed23d7 100644 --- a/tests/unit/wallet/test_database.py +++ b/tests/unit/wallet/test_database.py @@ -353,37 +353,37 @@ class TestQueries(AsyncioTestCase): self.assertListEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs]) self.assertListEqual([3, 2, 1], [tx.height for tx in txs]) - txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts) + txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts, include_is_my_output=True) self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs]) - self.assertEqual(txs[0].inputs[0].is_my_account, True) - self.assertEqual(txs[0].outputs[0].is_my_account, False) - self.assertEqual(txs[1].inputs[0].is_my_account, False) - self.assertEqual(txs[1].outputs[0].is_my_account, True) + self.assertEqual(txs[0].inputs[0].is_my_input, True) + self.assertEqual(txs[0].outputs[0].is_my_output, False) + self.assertEqual(txs[1].inputs[0].is_my_input, False) + self.assertEqual(txs[1].outputs[0].is_my_output, True) - txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2]) + txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2], include_is_my_output=True) self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs]) - self.assertEqual(txs[0].inputs[0].is_my_account, True) - self.assertEqual(txs[0].outputs[0].is_my_account, False) - self.assertEqual(txs[1].inputs[0].is_my_account, False) - self.assertEqual(txs[1].outputs[0].is_my_account, True) + self.assertEqual(txs[0].inputs[0].is_my_input, True) + self.assertEqual(txs[0].outputs[0].is_my_output, False) + self.assertEqual(txs[1].inputs[0].is_my_input, False) + self.assertEqual(txs[1].outputs[0].is_my_output, True) self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account2])) tx = await self.ledger.db.get_transaction(txid=tx2.id) self.assertEqual(tx.id, tx2.id) - self.assertFalse(tx.inputs[0].is_my_account) - self.assertFalse(tx.outputs[0].is_my_account) - tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id) - self.assertTrue(tx.inputs[0].is_my_account) - self.assertFalse(tx.outputs[0].is_my_account) - tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id) - self.assertFalse(tx.inputs[0].is_my_account) - self.assertTrue(tx.outputs[0].is_my_account) + self.assertIsNone(tx.inputs[0].is_my_input) + self.assertIsNone(tx.outputs[0].is_my_output) + tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id, include_is_my_output=True) + self.assertTrue(tx.inputs[0].is_my_input) + self.assertFalse(tx.outputs[0].is_my_output) + tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id, include_is_my_output=True) + self.assertFalse(tx.inputs[0].is_my_input) + self.assertTrue(tx.outputs[0].is_my_output) # height 0 sorted to the top with the rest in descending order tx4 = await self.create_tx_from_nothing(account1, 0) txos = await self.ledger.db.get_txos() - self.assertListEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos]) - self.assertListEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos]) + self.assertListEqual([0, 3, 2, 2, 1], [txo.tx_ref.height for txo in txos]) + self.assertListEqual([tx4.id, tx3.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos]) txs = await self.ledger.db.get_transactions(accounts=[account1, account2]) self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs]) self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs]) diff --git a/tests/unit/wallet/test_transaction.py b/tests/unit/wallet/test_transaction.py index a876a12b9..7c0942cd0 100644 --- a/tests/unit/wallet/test_transaction.py +++ b/tests/unit/wallet/test_transaction.py @@ -82,14 +82,14 @@ class TestSizeAndFeeEstimation(AsyncioTestCase): class TestAccountBalanceImpactFromTransaction(unittest.TestCase): - def test_is_my_account_not_set(self): + def test_is_my_output_not_set(self): tx = get_transaction() with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"): _ = tx.net_account_balance - tx.inputs[0].txo_ref.txo.is_my_account = True + tx.inputs[0].txo_ref.txo.is_my_output = True with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"): _ = tx.net_account_balance - tx.outputs[0].is_my_account = True + tx.outputs[0].is_my_output = True # all inputs/outputs are set now so it should work _ = tx.net_account_balance @@ -98,9 +98,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) - tx.inputs[0].txo_ref.txo.is_my_account = True - tx.outputs[0].is_my_account = False - tx.outputs[1].is_my_account = True + tx.inputs[0].txo_ref.txo.is_my_output = True + tx.outputs[0].is_my_output = False + tx.outputs[1].is_my_output = True self.assertEqual(tx.net_account_balance, -200*CENT) def test_paying_from_other_account_to_my_account(self): @@ -108,9 +108,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) - tx.inputs[0].txo_ref.txo.is_my_account = False - tx.outputs[0].is_my_account = True - tx.outputs[1].is_my_account = False + tx.inputs[0].txo_ref.txo.is_my_output = False + tx.outputs[0].is_my_output = True + tx.outputs[1].is_my_output = False self.assertEqual(tx.net_account_balance, 190*CENT) def test_paying_from_my_account_to_my_account(self): @@ -118,9 +118,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) - tx.inputs[0].txo_ref.txo.is_my_account = True - tx.outputs[0].is_my_account = True - tx.outputs[1].is_my_account = True + tx.inputs[0].txo_ref.txo.is_my_output = True + tx.outputs[0].is_my_output = True + tx.outputs[1].is_my_output = True self.assertEqual(tx.net_account_balance, -10*CENT) # lost to fee