From 7b86b3843f74cc4640cd1addaf1ef31fe53b83b7 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 25 Oct 2019 23:34:44 -0400 Subject: [PATCH] refactored and updates all list commands to always be paginated --- lbry/lbry/extras/daemon/Daemon.py | 158 ++++++++--------- lbry/lbry/testcase.py | 5 +- .../integration/test_account_commands.py | 57 ++++--- lbry/tests/integration/test_chris45.py | 4 +- lbry/tests/integration/test_claim_commands.py | 160 +++++++++--------- lbry/tests/integration/test_file_commands.py | 2 +- lbry/tests/integration/test_other_commands.py | 8 +- .../tests/integration/test_wallet_commands.py | 8 +- torba/torba/client/baseaccount.py | 1 + torba/torba/client/basedatabase.py | 2 +- torba/torba/client/wallet.py | 8 +- 11 files changed, 203 insertions(+), 210 deletions(-) diff --git a/lbry/lbry/extras/daemon/Daemon.py b/lbry/lbry/extras/daemon/Daemon.py index 3b71e7e22..3f155d55a 100644 --- a/lbry/lbry/extras/daemon/Daemon.py +++ b/lbry/lbry/extras/daemon/Daemon.py @@ -10,7 +10,7 @@ import random import ecdsa import hashlib from urllib.parse import urlencode, quote -from typing import Callable, Optional, List, Dict, Tuple +from typing import Callable, Optional, List from binascii import hexlify, unhexlify from traceback import format_exc from aiohttp import web @@ -111,7 +111,7 @@ CONNECTION_MESSAGES = { SHORT_ID_LEN = 20 MAX_UPDATE_FEE_ESTIMATE = 0.3 -DEFAULT_PAGE_SIZE = 50 +DEFAULT_PAGE_SIZE = 20 def encode_pagination_doc(items): @@ -124,33 +124,38 @@ def encode_pagination_doc(items): } -async def maybe_paginate(get_records_with_count: Callable[[bool, Dict], Tuple[List, int]], - page: Optional[int], page_size: Optional[int], - no_totals: Optional[bool] = None, **constraints): - if page is None and page_size is None: - return (await get_records_with_count(fetch_count=False, **constraints))[0] - if no_totals is not None: - constraints["no_totals"] = no_totals - if page is None: - page = 1 - if page_size is None or page_size > DEFAULT_PAGE_SIZE: - page_size = DEFAULT_PAGE_SIZE +async def paginate_rows(get_records: Callable, get_record_count: Callable, + page: Optional[int], page_size: Optional[int], **constraints): + page = max(1, page or 1) + page_size = max(1, page_size or DEFAULT_PAGE_SIZE) constraints.update({ "offset": page_size * (page - 1), "limit": page_size }) - fetch_count = not no_totals - items, total_items = await get_records_with_count(fetch_count, **constraints) - result = { + items = await get_records(**constraints) + total_items = await get_record_count(**constraints) + return { "items": items, + "total_pages": int((total_items + (page_size - 1)) / page_size), + "total_items": total_items, + "page": page, "page_size": page_size + } + + +def paginate_list(items: List, page: Optional[int], page_size: Optional[int]): + page = max(1, page or 1) + page_size = max(1, page_size or DEFAULT_PAGE_SIZE) + total_items = len(items) + offset = page_size * (page - 1) + subitems = [] + if offset <= total_items: + subitems = items[offset:page_size] + return { + "items": subitems, + "total_pages": int((total_items + (page_size - 1)) / page_size), + "total_items": total_items, "page": page, "page_size": page_size } - if not no_totals: - result.update({ - "total_pages": (total_items + (page_size - 1)) // page_size, - "total_items": total_items, - }) - return result def sort_claim_results(claims): @@ -1048,21 +1053,23 @@ class Daemon(metaclass=JSONRPCServerType): """ @requires("wallet") - def jsonrpc_wallet_list(self, wallet_id=None): + def jsonrpc_wallet_list(self, wallet_id=None, page=None, page_size=None): """ List wallets. Usage: - wallet_list [--wallet_id=] + wallet_list [--wallet_id=] [--page=] [--page_size=] Options: --wallet_id= : (str) show specific wallet only + --page= : (int) page to return during paginating + --page_size= : (int) number of items on page during pagination - Returns: {List[Wallet]} + Returns: {Paginated[Wallet]} """ if wallet_id: - return [self.wallet_manager.get_wallet_or_error(wallet_id)] - return self.wallet_manager.wallets + return paginate_list([self.wallet_manager.get_wallet_or_error(wallet_id)], 1, 1) + return paginate_list(self.wallet_manager.wallets, page, page_size) @requires("wallet") async def jsonrpc_wallet_create( @@ -1310,8 +1317,9 @@ class Daemon(metaclass=JSONRPCServerType): """ @requires("wallet") - def jsonrpc_account_list(self, account_id=None, wallet_id=None, confirmations=0, - include_claims=False, show_seed=False): + async def jsonrpc_account_list( + self, account_id=None, wallet_id=None, confirmations=0, + include_claims=False, show_seed=False, page=None, page_size=None): """ List details of all of the accounts or a specific account. @@ -1319,6 +1327,7 @@ class Daemon(metaclass=JSONRPCServerType): account_list [] [--wallet_id=] [--confirmations=] [--include_claims] [--show_seed] + [--page=] [--page_size=] Options: --account_id= : (str) If provided only the balance for this @@ -1328,8 +1337,10 @@ class Daemon(metaclass=JSONRPCServerType): --include_claims : (bool) include claims, requires than a LBC account is specified (default: false) --show_seed : (bool) show the seed for the account + --page= : (int) page to return during paginating + --page_size= : (int) number of items on page during pagination - Returns: {List[Account]} + Returns: {Paginated[Account]} """ kwargs = { 'confirmations': confirmations, @@ -1337,9 +1348,9 @@ class Daemon(metaclass=JSONRPCServerType): } wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if account_id: - return wallet.get_account_or_error(account_id).get_details(**kwargs) + return paginate_list([await wallet.get_account_or_error(account_id).get_details(**kwargs)], 1, 1) else: - return wallet.get_detailed_accounts(**kwargs) + return paginate_list(await wallet.get_detailed_accounts(**kwargs), page, page_size) @requires("wallet") async def jsonrpc_account_balance(self, account_id=None, wallet_id=None, confirmations=0, reserved_subtotals=False): @@ -1723,8 +1734,9 @@ class Daemon(metaclass=JSONRPCServerType): constraints['accounts'] = [wallet.get_account_or_error(account_id)] else: constraints['accounts'] = wallet.accounts - return maybe_paginate( - partial(get_records_with_count, self.ledger.get_addresses, self.ledger.get_address_count), + return paginate_rows( + self.ledger.get_addresses, + self.ledger.get_address_count, page, page_size, **constraints ) @@ -1952,13 +1964,10 @@ class Daemon(metaclass=JSONRPCServerType): else: claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts) claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts) - return maybe_paginate( - partial(get_records_with_count, claims, claim_count), - page, page_size - ) + return paginate_rows(claims, claim_count, page, page_size) @requires(WALLET_COMPONENT) - def jsonrpc_claim_search(self, page=None, page_size=None, no_totals=None, **kwargs): + async def jsonrpc_claim_search(self, **kwargs): """ Search for stream and channel claims on the blockchain. @@ -2083,18 +2092,18 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {Paginated[Output]} """ - async def get_claims_with_count(fetch_count, **kwargs): - claims, offset, count = await self.ledger.claim_search(**kwargs) - return (claims, count) - if kwargs.pop('valid_channel_signature', False): kwargs['signature_valid'] = 1 if kwargs.pop('invalid_channel_signature', False): kwargs['signature_valid'] = 0 - return maybe_paginate( - get_claims_with_count, - page, page_size, no_totals, **kwargs - ) + page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50) + kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size}) + txos, offset, total = await self.ledger.claim_search(**kwargs) + result = {"items": txos, "page": page_num, "page_size": page_size} + if not kwargs.pop('no_totals', False): + result['total_pages'] = int((total + (page_size - 1)) / page_size) + result['total_items'] = total + return result CHANNEL_DOC = """ Create, update, abandon and list your channel claims. @@ -2460,10 +2469,7 @@ class Daemon(metaclass=JSONRPCServerType): else: channels = partial(self.ledger.get_channels, wallet=wallet, accounts=wallet.accounts) channel_count = partial(self.ledger.get_channel_count, wallet=wallet, accounts=wallet.accounts) - return maybe_paginate( - partial(get_records_with_count, channels, channel_count), - page, page_size - ) + return paginate_rows(channels, channel_count, page, page_size) @requires(WALLET_COMPONENT) async def jsonrpc_channel_export(self, channel_id=None, channel_name=None, account_id=None, wallet_id=None): @@ -3118,10 +3124,7 @@ class Daemon(metaclass=JSONRPCServerType): else: streams = partial(self.ledger.get_streams, wallet=wallet, accounts=wallet.accounts) stream_count = partial(self.ledger.get_stream_count, wallet=wallet, accounts=wallet.accounts) - return maybe_paginate( - partial(get_records_with_count, streams, stream_count), - page, page_size - ) + return paginate_rows(streams, stream_count, page, page_size) @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DHT_COMPONENT, DATABASE_COMPONENT) @@ -3224,10 +3227,7 @@ class Daemon(metaclass=JSONRPCServerType): else: supports = partial(self.ledger.get_supports, wallet=wallet, accounts=wallet.accounts) support_count = partial(self.ledger.get_support_count, wallet=wallet, accounts=wallet.accounts) - return maybe_paginate( - partial(get_records_with_count, supports, support_count), - page, page_size - ) + return paginate_rows(supports, support_count, page, page_size) @requires(WALLET_COMPONENT) async def jsonrpc_support_abandon( @@ -3377,10 +3377,7 @@ class Daemon(metaclass=JSONRPCServerType): self.ledger.get_transaction_history, wallet=wallet, accounts=wallet.accounts) transaction_count = partial( self.ledger.get_transaction_history_count, wallet=wallet, accounts=wallet.accounts) - return maybe_paginate( - partial(get_records_with_count, transactions, transaction_count), - page, page_size - ) + return paginate_rows(transactions, transaction_count, page, page_size) @requires(WALLET_COMPONENT) def jsonrpc_transaction_show(self, txid): @@ -3426,10 +3423,7 @@ class Daemon(metaclass=JSONRPCServerType): else: utxos = partial(self.ledger.get_utxos, wallet=wallet, accounts=wallet.accounts) utxo_count = partial(self.ledger.get_utxo_count, wallet=wallet, accounts=wallet.accounts) - return maybe_paginate( - partial(get_records_with_count, utxos, utxo_count), - page, page_size - ) + return paginate_rows(utxos, utxo_count, page, page_size) @requires(WALLET_COMPONENT) async def jsonrpc_utxo_release(self, account_id=None, wallet_id=None): @@ -3514,19 +3508,22 @@ class Daemon(metaclass=JSONRPCServerType): """ @requires(DHT_COMPONENT) - async def jsonrpc_peer_list(self, blob_hash, search_bottom_out_limit=None): + async def jsonrpc_peer_list(self, blob_hash, search_bottom_out_limit=None, page=None, page_size=None): """ Get peers for blob hash Usage: peer_list ( | --blob_hash=) - [ | --search_bottom_out_limit=] + [ | --search_bottom_out_limit=] + [--page=] [--page_size=] Options: --blob_hash= : (str) find available peers for this blob hash --search_bottom_out_limit= : (int) the number of search probes in a row that don't find any new peers before giving up and returning + --page= : (int) page to return during paginating + --page_size= : (int) number of items on page during pagination Returns: (list) List of contact dictionaries {'address': , 'udp_port': , 'tcp_port': , @@ -3555,7 +3552,7 @@ class Daemon(metaclass=JSONRPCServerType): } for peer in peers ] - return results + return paginate_list(results, page, page_size) @requires(DATABASE_COMPONENT) async def jsonrpc_blob_announce(self, blob_hash=None, stream_hash=None, sd_hash=None): @@ -3593,7 +3590,7 @@ class Daemon(metaclass=JSONRPCServerType): @requires(BLOB_COMPONENT, WALLET_COMPONENT) async def jsonrpc_blob_list(self, uri=None, stream_hash=None, sd_hash=None, needed=None, - finished=None, page_size=None, page=None): + finished=None, page=None, page_size=None): """ Returns blob hashes. If not given filters, returns all blobs known by the blob manager @@ -3601,8 +3598,7 @@ class Daemon(metaclass=JSONRPCServerType): blob_list [--needed] [--finished] [ | --uri=] [ | --stream_hash=] [ | --sd_hash=] - [ | --page_size=] - [ | --page=] + [--page=] [--page_size=] Options: --needed : (bool) only return needed blobs @@ -3610,8 +3606,8 @@ class Daemon(metaclass=JSONRPCServerType): --uri= : (str) filter blobs by stream in a uri --stream_hash= : (str) filter blobs by stream hash --sd_hash= : (str) filter blobs by sd hash - --page_size= : (int) results page size - --page= : (int) page of results to return + --page= : (int) page to return during paginating + --page_size= : (int) number of items on page during pagination Returns: (list) List of blob hashes @@ -3639,11 +3635,7 @@ class Daemon(metaclass=JSONRPCServerType): blobs = [blob_hash for blob_hash in blobs if not self.blob_manager.is_blob_verified(blob_hash)] if finished: blobs = [blob_hash for blob_hash in blobs if self.blob_manager.is_blob_verified(blob_hash)] - page_size = page_size or len(blobs) - page = page or 0 - start_index = page * page_size - stop_index = start_index + page_size - return blobs[start_index:stop_index] + return paginate_list(blobs, page, page_size) @requires(BLOB_COMPONENT) async def jsonrpc_blob_reflect(self, blob_hashes, reflector_server=None): @@ -4153,9 +4145,3 @@ def get_loggly_query_string(installation_id): } data = urlencode(params) return base_loggly_search_url + data - - -async def get_records_with_count(get_records, get_record_count, fetch_count, **kwargs): - records = await get_records(**kwargs) - record_count = await get_record_count(**kwargs) if fetch_count else 0 - return (records, record_count) diff --git a/lbry/lbry/testcase.py b/lbry/lbry/testcase.py index f16e0c41b..afab42765 100644 --- a/lbry/lbry/testcase.py +++ b/lbry/lbry/testcase.py @@ -276,8 +276,11 @@ class CommandTestCase(IntegrationTestCase): return await self.out(self.daemon.jsonrpc_resolve(uri)) async def claim_search(self, **kwargs): - return await self.out(self.daemon.jsonrpc_claim_search(**kwargs)) + return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items'] @staticmethod def get_claim_id(tx): return tx['outputs'][0]['claim_id'] + + def assertItemCount(self, result, count): + self.assertEqual(result['total_items'], count) diff --git a/lbry/tests/integration/test_account_commands.py b/lbry/tests/integration/test_account_commands.py index 2c5fbbbe8..fab3958dc 100644 --- a/lbry/tests/integration/test_account_commands.py +++ b/lbry/tests/integration/test_account_commands.py @@ -13,55 +13,55 @@ def extract(d, keys): class AccountManagement(CommandTestCase): async def test_account_list_set_create_remove_add(self): # check initial account - response = await self.daemon.jsonrpc_account_list() - self.assertEqual(len(response['lbc_regtest']), 1) + accounts = await self.daemon.jsonrpc_account_list() + self.assertItemCount(accounts, 1) # change account name and gap - account_id = response['lbc_regtest'][0]['id'] + account_id = accounts['items'][0]['id'] self.daemon.jsonrpc_account_set( account_id=account_id, new_name='test account', receiving_gap=95, receiving_max_uses=96, change_gap=97, change_max_uses=98 ) - response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0] - self.assertEqual(response['name'], 'test account') + accounts = (await self.daemon.jsonrpc_account_list())['items'][0] + self.assertEqual(accounts['name'], 'test account') self.assertEqual( - response['address_generator']['receiving'], + accounts['address_generator']['receiving'], {'gap': 95, 'maximum_uses_per_address': 96} ) self.assertEqual( - response['address_generator']['change'], + accounts['address_generator']['change'], {'gap': 97, 'maximum_uses_per_address': 98} ) # create another account await self.daemon.jsonrpc_account_create('second account') - response = await self.daemon.jsonrpc_account_list() - self.assertEqual(len(response['lbc_regtest']), 2) - self.assertEqual(response['lbc_regtest'][1]['name'], 'second account') - account_id2 = response['lbc_regtest'][1]['id'] + accounts = await self.daemon.jsonrpc_account_list() + self.assertItemCount(accounts, 2) + self.assertEqual(accounts['items'][1]['name'], 'second account') + account_id2 = accounts['items'][1]['id'] # make new account the default self.daemon.jsonrpc_account_set(account_id=account_id2, default=True) - response = await self.daemon.jsonrpc_account_list(show_seed=True) - self.assertEqual(response['lbc_regtest'][0]['name'], 'second account') + accounts = await self.daemon.jsonrpc_account_list(show_seed=True) + self.assertEqual(accounts['items'][0]['name'], 'second account') - account_seed = response['lbc_regtest'][1]['seed'] + account_seed = accounts['items'][1]['seed'] # remove account - self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id']) - response = await self.daemon.jsonrpc_account_list() - self.assertEqual(len(response['lbc_regtest']), 1) + self.daemon.jsonrpc_account_remove(accounts['items'][1]['id']) + accounts = await self.daemon.jsonrpc_account_list() + self.assertItemCount(accounts, 1) # add account await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed) - response = await self.daemon.jsonrpc_account_list() - self.assertEqual(len(response['lbc_regtest']), 2) - self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account') + accounts = await self.daemon.jsonrpc_account_list() + self.assertItemCount(accounts, 2) + self.assertEqual(accounts['items'][1]['name'], 'recreated account') # list specific account - response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True) - self.assertEqual(response['name'], 'recreated account') + accounts = await self.daemon.jsonrpc_account_list(account_id, include_claims=True) + self.assertEqual(accounts['items'][0]['name'], 'recreated account') async def test_wallet_migration(self): # null certificates should get deleted @@ -75,10 +75,10 @@ class AccountManagement(CommandTestCase): self.assertEqual(list(self.account.channel_keys.keys()), [keys[2]]) async def assertFindsClaims(self, claim_names, awaitable): - self.assertEqual(claim_names, [txo.claim_name for txo in await awaitable]) + self.assertEqual(claim_names, [txo.claim_name for txo in (await awaitable)['items']]) async def assertOutputAmount(self, amounts, awaitable): - self.assertEqual(amounts, [dewies_to_lbc(txo.amount) for txo in await awaitable]) + self.assertEqual(amounts, [dewies_to_lbc(txo.amount) for txo in (await awaitable)['items']]) async def test_commands_across_accounts(self): channel_list = self.daemon.jsonrpc_channel_list @@ -131,12 +131,13 @@ class AccountManagement(CommandTestCase): support2 = await self.support_create( self.get_claim_id(stream2), '0.01', account_id=second_account.id, funding_account_ids=[default_account.id] ) - self.assertEqual([support2['txid'], support1['txid']], [txo.tx_ref.id for txo in await support_list()]) - self.assertEqual([support1['txid']], [txo.tx_ref.id for txo in await support_list(account_id=default_account.id)]) - self.assertEqual([support2['txid']], [txo.tx_ref.id for txo in await support_list(account_id=second_account.id)]) + self.assertEqual([support2['txid'], support1['txid']], [txo.tx_ref.id for txo in (await support_list())['items']]) + self.assertEqual([support1['txid']], [txo.tx_ref.id for txo in (await support_list(account_id=default_account.id))['items']]) + self.assertEqual([support2['txid']], [txo.tx_ref.id for txo in (await support_list(account_id=second_account.id))['items']]) history = await self.daemon.jsonrpc_transaction_list() - self.assertEqual(len(history), 8) + self.assertItemCount(history, 8) + history = history['items'] self.assertEqual(extract(history[0]['support_info'][0], ['claim_name', 'is_tip', 'amount', 'balance_delta']), { 'claim_name': 'stream-in-account2', 'is_tip': False, diff --git a/lbry/tests/integration/test_chris45.py b/lbry/tests/integration/test_chris45.py index 88cf5746f..1f7813e02 100644 --- a/lbry/tests/integration/test_chris45.py +++ b/lbry/tests/integration/test_chris45.py @@ -23,8 +23,8 @@ class EpicAdventuresOfChris45(CommandTestCase): # Do we have it locally? channels = await self.out(self.daemon.jsonrpc_channel_list()) - self.assertEqual(len(channels), 1) - self.assertEqual(channels[0]['name'], '@spam') + self.assertItemCount(channels, 1) + self.assertEqual(channels['items'][0]['name'], '@spam') # As the new channel claim travels through the intertubes and makes its # way into the mempool and then a block and then into the claimtrie, diff --git a/lbry/tests/integration/test_claim_commands.py b/lbry/tests/integration/test_claim_commands.py index b6e05744f..5fc72062d 100644 --- a/lbry/tests/integration/test_claim_commands.py +++ b/lbry/tests/integration/test_claim_commands.py @@ -43,20 +43,20 @@ class ClaimSearchCommand(ClaimTestCase): self.channel = await self.channel_create('@abc', '1.0') self.channel_id = self.get_claim_id(self.channel) - async def create_lots_of_streams(self, claims=4, blocks=3): + async def create_lots_of_streams(self): tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True) await self.confirm_tx(tx.id) - # 4 claims per block, 3 blocks (by default). Sorted by height (descending) then claim name (ascending). + # 4 claims per block, 3 blocks. Sorted by height (descending) then claim name (ascending). self.streams = [] - for j in range(blocks): + for j in range(4): same_height_claims = [] - for k in range(claims - 1): + for k in range(5): claim_tx = await self.stream_create( f'c{j}-{k}', '0.000001', channel_id=self.channel_id, confirm=False) same_height_claims.append(claim_tx['outputs'][0]['name']) await self.on_transaction_dict(claim_tx) claim_tx = await self.stream_create( - f'c{j}-{claims - 1}', '0.000001', channel_id=self.channel_id, confirm=True) + f'c{j}-6', '0.000001', channel_id=self.channel_id, confirm=True) same_height_claims.append(claim_tx['outputs'][0]['name']) self.streams = same_height_claims + self.streams @@ -139,40 +139,44 @@ class ClaimSearchCommand(ClaimTestCase): async def test_pagination(self): await self.create_channel() - await self.create_lots_of_streams(10, 10) + await self.create_lots_of_streams() - page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name']) - page_claim_ids = [item['name'] for item in page['items']] - self.assertEqual(page_claim_ids, self.streams[:20]) - - page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name']) - page_claim_ids = [item['name'] for item in page['items']] - self.assertEqual(page_claim_ids, self.streams[:6]) - - page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name']) - page_claim_ids = [item['name'] for item in page['items']] - self.assertEqual(page_claim_ids, self.streams[6:(2 * 6)]) - - page = await self.claim_search(page=1, channel='@abc', order_by=['height', '^name']) - page_claim_ids = [item['name'] for item in page['items']] - self.assertEqual(page_claim_ids, self.streams[:DEFAULT_PAGE_SIZE]) - - page = await self.claim_search(page=2, channel='@abc', order_by=['height', '^name']) - page_claim_ids = [item['name'] for item in page['items']] - self.assertEqual(page_claim_ids, self.streams[DEFAULT_PAGE_SIZE:(2 * DEFAULT_PAGE_SIZE)]) - - out_of_bounds = await self.claim_search(page=20, page_size=20, channel='@abc') - self.assertEqual(out_of_bounds['items'], []) - - total_claims = 10 * 10 + 1 - results = await self.claim_search(page=1) - self.assertEqual(results['total_pages'], (total_claims + DEFAULT_PAGE_SIZE - 1) // DEFAULT_PAGE_SIZE) - self.assertEqual(results['total_items'], total_claims) - - results = await self.claim_search(page=1, no_totals=True) + # with and without totals + results = await self.daemon.jsonrpc_claim_search() + self.assertEqual(results['total_pages'], 2) + self.assertEqual(results['total_items'], 25) + results = await self.daemon.jsonrpc_claim_search(no_totals=True) self.assertNotIn('total_pages', results) self.assertNotIn('total_items', results) + # defaults + page = await self.claim_search(channel='@abc', order_by=['height', '^name']) + page_claim_ids = [item['name'] for item in page] + self.assertEqual(page_claim_ids, self.streams[:DEFAULT_PAGE_SIZE]) + + # page with default page_size + page = await self.claim_search(page=2, channel='@abc', order_by=['height', '^name']) + page_claim_ids = [item['name'] for item in page] + self.assertEqual(page_claim_ids, self.streams[DEFAULT_PAGE_SIZE:(DEFAULT_PAGE_SIZE*2)]) + + # page_size larger than dataset + page = await self.claim_search(page_size=50, channel='@abc', order_by=['height', '^name']) + page_claim_ids = [item['name'] for item in page] + self.assertEqual(page_claim_ids, self.streams) + + # page_size less than dataset + page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name']) + page_claim_ids = [item['name'] for item in page] + self.assertEqual(page_claim_ids, self.streams[:6]) + + # page and page_size + page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name']) + page_claim_ids = [item['name'] for item in page] + self.assertEqual(page_claim_ids, self.streams[6:12]) + + out_of_bounds = await self.claim_search(page=4, page_size=20, channel='@abc') + self.assertEqual(out_of_bounds, []) + async def test_tag_search(self): claim1 = await self.stream_create('claim1', tags=['aBc']) claim2 = await self.stream_create('claim2', tags=['#abc', 'def']) @@ -330,7 +334,7 @@ class ChannelCommands(CommandTestCase): async def test_create_channel_names(self): # claim new name await self.channel_create('@foo') - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '8.991893') # fail to claim duplicate @@ -342,19 +346,19 @@ class ChannelCommands(CommandTestCase): await self.channel_create('foo') # nothing's changed after failed attempts - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '8.991893') # succeed overriding duplicate restriction await self.channel_create('@foo', allow_duplicate_name=True) - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 2) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 2) await self.assertBalance(self.account, '7.983786') async def test_channel_bids(self): # enough funds tx = await self.channel_create('@foo', '5.0') claim_id = self.get_claim_id(tx) - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '4.991893') # bid preserved on update @@ -371,14 +375,14 @@ class ChannelCommands(CommandTestCase): with self.assertRaisesRegex( InsufficientFundsError, "Not enough funds to cover this transaction."): await self.channel_create('@foo2', '9.0') - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '5.991447') # spend exactly amount available, no change tx = await self.channel_create('@foo3', '5.981266') await self.assertBalance(self.account, '0.0') self.assertEqual(len(tx['outputs']), 1) # no change - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 2) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 2) async def test_setting_channel_fields(self): values = { @@ -455,16 +459,16 @@ class ChannelCommands(CommandTestCase): account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id']) # before moving - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3) - self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 0) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 3) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 0) other_address = await account2.receiving.get_or_create_usable_address() tx = await self.out(self.channel_update(claim_id, claim_address=other_address)) # after moving - self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3) - self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=self.account.id)), 2) - self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 1) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 3) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=self.account.id), 2) + self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 1) async def test_channel_export_import_before_sending_channel(self): # export @@ -475,9 +479,9 @@ class ChannelCommands(CommandTestCase): # import daemon2 = await self.add_daemon() - self.assertEqual(0, len(await daemon2.jsonrpc_channel_list())) + self.assertItemCount(await daemon2.jsonrpc_channel_list(), 0) await daemon2.jsonrpc_channel_import(exported_data) - channels = await daemon2.jsonrpc_channel_list() + channels = (await daemon2.jsonrpc_channel_list())['items'] self.assertEqual(1, len(channels)) self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string()) @@ -499,11 +503,11 @@ class ChannelCommands(CommandTestCase): await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=self.account.id) # channel is in account2 await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=account2.id) - result = await self.out(self.daemon.jsonrpc_channel_list()) + result = (await self.out(self.daemon.jsonrpc_channel_list()))['items'] self.assertEqual(result[0]['amount'], '2.0') # check all accounts for channel await self.channel_update(self.get_claim_id(channel), bid='3.0') - result = await self.out(self.daemon.jsonrpc_channel_list()) + result = (await self.out(self.daemon.jsonrpc_channel_list()))['items'] self.assertEqual(result[0]['amount'], '3.0') await self.channel_abandon(self.get_claim_id(channel)) @@ -527,7 +531,7 @@ class StreamCommands(ClaimTestCase): async def test_create_stream_names(self): # claim new name await self.stream_create('foo') - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '8.993893') # fail to claim duplicate @@ -540,19 +544,19 @@ class StreamCommands(ClaimTestCase): Exception, "Stream names cannot start with '@' symbol."): await self.stream_create('@foo') - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '8.993893') # succeed overriding duplicate restriction await self.stream_create('foo', allow_duplicate_name=True) - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 2) await self.assertBalance(self.account, '7.987786') async def test_stream_bids(self): # enough funds tx = await self.stream_create('foo', '2.0') claim_id = self.get_claim_id(tx) - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '7.993893') # bid preserved on update @@ -569,14 +573,14 @@ class StreamCommands(ClaimTestCase): with self.assertRaisesRegex( InsufficientFundsError, "Not enough funds to cover this transaction."): await self.stream_create('foo2', '9.0') - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '6.993319') # spend exactly amount available, no change tx = await self.stream_create('foo3', '6.98523') await self.assertBalance(self.account, '0.0') self.assertEqual(len(tx['outputs']), 1) # no change - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 2) async def test_stream_update_and_abandon_across_accounts(self): account2 = await self.daemon.jsonrpc_account_create('second account') @@ -586,11 +590,11 @@ class StreamCommands(ClaimTestCase): await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=self.account.id) # stream is in account2 await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=account2.id) - result = await self.out(self.daemon.jsonrpc_stream_list()) + result = (await self.out(self.daemon.jsonrpc_stream_list()))['items'] self.assertEqual(result[0]['amount'], '2.0') # check all accounts for stream await self.stream_update(self.get_claim_id(stream), bid='3.0') - result = await self.out(self.daemon.jsonrpc_stream_list()) + result = (await self.out(self.daemon.jsonrpc_stream_list()))['items'] self.assertEqual(result[0]['amount'], '3.0') await self.stream_abandon(self.get_claim_id(stream)) @@ -614,18 +618,18 @@ class StreamCommands(ClaimTestCase): baz_id = self.get_claim_id(baz_tx) channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id)) - self.assertEqual(len(channels), 1) - self.assertEqual(channels[0]['name'], '@spam') + self.assertItemCount(channels, 1) + self.assertEqual(channels['items'][0]['name'], '@spam') self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list(account1_id))) channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id)) - self.assertEqual(len(channels), 1) - self.assertEqual(channels[0]['name'], '@baz') + self.assertItemCount(channels, 1) + self.assertEqual(channels['items'][0]['name'], '@baz') channels = await self.out(self.daemon.jsonrpc_channel_list()) - self.assertEqual(len(channels), 2) - self.assertEqual(channels[0]['name'], '@baz') - self.assertEqual(channels[1]['name'], '@spam') + self.assertItemCount(channels, 2) + self.assertEqual(channels['items'][0]['name'], '@baz') + self.assertEqual(channels['items'][1]['name'], '@spam') # defaults to using all accounts to lookup channel await self.stream_create('hovercraft1', '0.1', channel_id=baz_id) @@ -817,17 +821,17 @@ class StreamCommands(ClaimTestCase): account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id']) # before sending - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4) - self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=self.account.id)), 4) - self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 0) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 4) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=self.account.id), 4) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=account2_id), 0) other_address = await account2.receiving.get_or_create_usable_address() tx = await self.out(self.stream_update(claim_id, claim_address=other_address)) # after sending - self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4) - self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=self.account.id)), 3) - self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 4) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=self.account.id), 3) + self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=account2_id), 1) async def test_setting_fee_fields(self): tx = await self.out(self.stream_create('paid-stream')) @@ -1043,7 +1047,7 @@ class StreamCommands(ClaimTestCase): tx = await self.stream_create(bid='2.5') # creates new claim claim_id = self.get_claim_id(tx) - txs = await self.out(self.daemon.jsonrpc_transaction_list()) + txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items'] self.assertEqual(len(txs[0]['claim_info']), 1) self.assertEqual(txs[0]['confirmations'], 1) self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5') @@ -1057,7 +1061,7 @@ class StreamCommands(ClaimTestCase): self.assertEqual(0, len(self.daemon.jsonrpc_file_list())) await self.stream_update(claim_id, bid='1.0') # updates previous claim - txs = await self.out(self.daemon.jsonrpc_transaction_list()) + txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items'] self.assertEqual(len(txs[0]['update_info']), 1) self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5') self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id) @@ -1066,7 +1070,7 @@ class StreamCommands(ClaimTestCase): await self.assertBalance(self.account, '8.9796765') await self.stream_abandon(claim_id) - txs = await self.out(self.daemon.jsonrpc_transaction_list()) + txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items'] self.assertEqual(len(txs[0]['abandon_info']), 1) self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0') self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id) @@ -1169,7 +1173,7 @@ class SupportCommands(CommandTestCase): await self.assertBalance(account2, '3.9998585') # verify that the incoming tip is marked correctly as is_tip=True in account1 - txs = await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)) + txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items'] self.assertEqual(len(txs[0]['support_info']), 1) self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0') self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id) @@ -1178,9 +1182,9 @@ class SupportCommands(CommandTestCase): self.assertEqual(txs[0]['fee'], '0.0') # verify that the outgoing tip is marked correctly as is_tip=True in account2 - txs2 = await self.out( + txs2 = (await self.out( self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id) - ) + ))['items'] self.assertEqual(len(txs2[0]['support_info']), 1) self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0') self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id) @@ -1200,7 +1204,7 @@ class SupportCommands(CommandTestCase): await self.assertBalance(account2, '1.999717') # verify that the outgoing support is marked correctly as is_tip=False in account2 - txs2 = await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')) + txs2 = (await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')))['items'] self.assertEqual(len(txs2[0]['support_info']), 1) self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0') self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id) diff --git a/lbry/tests/integration/test_file_commands.py b/lbry/tests/integration/test_file_commands.py index 76836882c..9279dfa5b 100644 --- a/lbry/tests/integration/test_file_commands.py +++ b/lbry/tests/integration/test_file_commands.py @@ -263,7 +263,7 @@ class FileCommands(CommandTestCase): BlobDownloader.BAN_FACTOR = .5 # fixme: temporary field, will move to connection manager or a conf tx = await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23))) sd_hash = tx['outputs'][0]['value']['source']['sd_hash'] - missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2] + missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))['items'][-2] await self.daemon.jsonrpc_file_delete(claim_name='foo') # backup blob missing_blob = self.server_blob_manager.get_blob(missing_blob_hash) diff --git a/lbry/tests/integration/test_other_commands.py b/lbry/tests/integration/test_other_commands.py index 1a4b27322..2d93baeb2 100644 --- a/lbry/tests/integration/test_other_commands.py +++ b/lbry/tests/integration/test_other_commands.py @@ -5,8 +5,8 @@ class AddressManagement(CommandTestCase): async def test_address_list(self): addresses = await self.out(self.daemon.jsonrpc_address_list()) - self.assertEqual(27, len(addresses)) + self.assertItemCount(addresses, 27) - single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address'])) - self.assertEqual(1, len(single)) - self.assertEqual(single[0], addresses[11]) + single = await self.out(self.daemon.jsonrpc_address_list(addresses['items'][11]['address'])) + self.assertItemCount(single, 1) + self.assertEqual(single['items'][0], addresses['items'][11]) diff --git a/lbry/tests/integration/test_wallet_commands.py b/lbry/tests/integration/test_wallet_commands.py index bd44044d4..9a555c0ad 100644 --- a/lbry/tests/integration/test_wallet_commands.py +++ b/lbry/tests/integration/test_wallet_commands.py @@ -49,12 +49,12 @@ class WalletEncryptionAndSynchronization(CommandTestCase): "two": "2", "conflict": "2", "another": "A" }) - self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 1) + self.assertItemCount(await daemon.jsonrpc_account_list(), 1) data = await daemon2.jsonrpc_sync_apply('password') await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True) - self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 2) + self.assertItemCount(await daemon.jsonrpc_account_list(), 2) self.assertDictEqual( # "two" key added and "conflict" value changed to "2" daemon.jsonrpc_preference_get(), @@ -66,9 +66,9 @@ class WalletEncryptionAndSynchronization(CommandTestCase): await self.confirm_tx(channel.id, self.daemon2.ledger) # both daemons will have the channel but only one has the cert so far - self.assertEqual(len(await daemon.jsonrpc_channel_list()), 1) + self.assertItemCount(await daemon.jsonrpc_channel_list(), 1) self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0) - self.assertEqual(len(await daemon2.jsonrpc_channel_list()), 1) + self.assertItemCount(await daemon2.jsonrpc_channel_list(), 1) self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1) data = await daemon2.jsonrpc_sync_apply('password') diff --git a/torba/torba/client/baseaccount.py b/torba/torba/client/baseaccount.py index d6171a999..df063b594 100644 --- a/torba/torba/client/baseaccount.py +++ b/torba/torba/client/baseaccount.py @@ -334,6 +334,7 @@ class BaseAccount: details = { 'id': self.id, 'name': self.name, + 'ledger': self.ledger.get_id(), 'coins': round(satoshis/COIN, 2), 'satoshis': satoshis, 'encrypted': self.encrypted, diff --git a/torba/torba/client/basedatabase.py b/torba/torba/client/basedatabase.py index d4065e2c7..bb095a6c5 100644 --- a/torba/torba/client/basedatabase.py +++ b/torba/torba/client/basedatabase.py @@ -612,7 +612,7 @@ class BaseDatabase(SQLiteMixin): ) return addresses - async def get_address_count(self, **constraints): + async def get_address_count(self, cols=None, **constraints): count = await self.select_addresses('count(*)', **constraints) return count[0][0] diff --git a/torba/torba/client/wallet.py b/torba/torba/client/wallet.py index 185d9d456..7d27315cf 100644 --- a/torba/torba/client/wallet.py +++ b/torba/torba/client/wallet.py @@ -109,14 +109,12 @@ class Wallet: ] if account_ids else self.accounts async def get_detailed_accounts(self, **kwargs): - ledgers = {} + accounts = [] for i, account in enumerate(self.accounts): details = await account.get_details(**kwargs) details['is_default'] = i == 0 - ledger_id = account.ledger.get_id() - ledgers.setdefault(ledger_id, []) - ledgers[ledger_id].append(details) - return ledgers + accounts.append(details) + return accounts @classmethod def from_storage(cls, storage: 'WalletStorage', manager: 'basemanager.BaseWalletManager') -> 'Wallet':