refactored and updates all list commands to always be paginated

lbry/lbry/extras/daemon/Daemon.py

@@ -10,7 +10,7 @@ import random
 import ecdsa
 import hashlib
 from urllib.parse import urlencode, quote
-from typing import Callable, Optional, List, Dict, Tuple
+from typing import Callable, Optional, List
 from binascii import hexlify, unhexlify
 from traceback import format_exc
 from aiohttp import web
@@ -111,7 +111,7 @@ CONNECTION_MESSAGES = {
 
 SHORT_ID_LEN = 20
 MAX_UPDATE_FEE_ESTIMATE = 0.3
-DEFAULT_PAGE_SIZE = 50
+DEFAULT_PAGE_SIZE = 20
 
 
 def encode_pagination_doc(items):
@@ -124,33 +124,38 @@ def encode_pagination_doc(items):
     }
 
 
-async def maybe_paginate(get_records_with_count: Callable[[bool, Dict], Tuple[List, int]],
+async def paginate_rows(get_records: Callable, get_record_count: Callable,
-                         page: Optional[int], page_size: Optional[int],
+                        page: Optional[int], page_size: Optional[int], **constraints):
-                         no_totals: Optional[bool] = None, **constraints):
+    page = max(1, page or 1)
-    if page is None and page_size is None:
+    page_size = max(1, page_size or DEFAULT_PAGE_SIZE)
-        return (await get_records_with_count(fetch_count=False, **constraints))[0]
-    if no_totals is not None:
-        constraints["no_totals"] = no_totals
-    if page is None:
-        page = 1
-    if page_size is None or page_size > DEFAULT_PAGE_SIZE:
-        page_size = DEFAULT_PAGE_SIZE
     constraints.update({
         "offset": page_size * (page - 1),
         "limit": page_size
     })
-    fetch_count = not no_totals
+    items = await get_records(**constraints)
-    items, total_items = await get_records_with_count(fetch_count, **constraints)
+    total_items = await get_record_count(**constraints)
-    result = {
+    return {
         "items": items,
+        "total_pages": int((total_items + (page_size - 1)) / page_size),
+        "total_items": total_items,
+        "page": page, "page_size": page_size
+    }
+
+
+def paginate_list(items: List, page: Optional[int], page_size: Optional[int]):
+    page = max(1, page or 1)
+    page_size = max(1, page_size or DEFAULT_PAGE_SIZE)
+    total_items = len(items)
+    offset = page_size * (page - 1)
+    subitems = []
+    if offset <= total_items:
+        subitems = items[offset:page_size]
+    return {
+        "items": subitems,
+        "total_pages": int((total_items + (page_size - 1)) / page_size),
+        "total_items": total_items,
         "page": page, "page_size": page_size
     }
-    if not no_totals:
-        result.update({
-            "total_pages": (total_items + (page_size - 1)) // page_size,
-            "total_items": total_items,
-        })
-    return result
 
 
 def sort_claim_results(claims):
@@ -1048,21 +1053,23 @@ class Daemon(metaclass=JSONRPCServerType):
     """
 
     @requires("wallet")
-    def jsonrpc_wallet_list(self, wallet_id=None):
+    def jsonrpc_wallet_list(self, wallet_id=None, page=None, page_size=None):
         """
         List wallets.
 
         Usage:
-            wallet_list [--wallet_id=<wallet_id>]
+            wallet_list [--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>]
 
         Options:
             --wallet_id=<wallet_id>  : (str) show specific wallet only
+            --page=<page>            : (int) page to return during paginating
+            --page_size=<page_size>  : (int) number of items on page during pagination
 
-        Returns: {List[Wallet]}
+        Returns: {Paginated[Wallet]}
         """
         if wallet_id:
-            return [self.wallet_manager.get_wallet_or_error(wallet_id)]
+            return paginate_list([self.wallet_manager.get_wallet_or_error(wallet_id)], 1, 1)
-        return self.wallet_manager.wallets
+        return paginate_list(self.wallet_manager.wallets, page, page_size)
 
     @requires("wallet")
     async def jsonrpc_wallet_create(
@@ -1310,8 +1317,9 @@ class Daemon(metaclass=JSONRPCServerType):
     """
 
     @requires("wallet")
-    def jsonrpc_account_list(self, account_id=None, wallet_id=None, confirmations=0,
+    async def jsonrpc_account_list(
-                             include_claims=False, show_seed=False):
+            self, account_id=None, wallet_id=None, confirmations=0,
+            include_claims=False, show_seed=False, page=None, page_size=None):
         """
         List details of all of the accounts or a specific account.
 
@@ -1319,6 +1327,7 @@ class Daemon(metaclass=JSONRPCServerType):
             account_list [<account_id>] [--wallet_id=<wallet_id>]
                          [--confirmations=<confirmations>]
                          [--include_claims] [--show_seed]
+                         [--page=<page>] [--page_size=<page_size>]
 
         Options:
             --account_id=<account_id>       : (str) If provided only the balance for this
@@ -1328,8 +1337,10 @@ class Daemon(metaclass=JSONRPCServerType):
             --include_claims                : (bool) include claims, requires than a
                                                      LBC account is specified (default: false)
             --show_seed                     : (bool) show the seed for the account
+            --page=<page>                   : (int) page to return during paginating
+            --page_size=<page_size>         : (int) number of items on page during pagination
 
-        Returns: {List[Account]}
+        Returns: {Paginated[Account]}
         """
         kwargs = {
             'confirmations': confirmations,
@@ -1337,9 +1348,9 @@ class Daemon(metaclass=JSONRPCServerType):
         }
         wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
         if account_id:
-            return wallet.get_account_or_error(account_id).get_details(**kwargs)
+            return paginate_list([await wallet.get_account_or_error(account_id).get_details(**kwargs)], 1, 1)
         else:
-            return wallet.get_detailed_accounts(**kwargs)
+            return paginate_list(await wallet.get_detailed_accounts(**kwargs), page, page_size)
 
     @requires("wallet")
     async def jsonrpc_account_balance(self, account_id=None, wallet_id=None, confirmations=0, reserved_subtotals=False):
@@ -1723,8 +1734,9 @@ class Daemon(metaclass=JSONRPCServerType):
             constraints['accounts'] = [wallet.get_account_or_error(account_id)]
         else:
             constraints['accounts'] = wallet.accounts
-        return maybe_paginate(
+        return paginate_rows(
-            partial(get_records_with_count, self.ledger.get_addresses, self.ledger.get_address_count),
+            self.ledger.get_addresses,
+            self.ledger.get_address_count,
             page, page_size, **constraints
         )
 
@@ -1952,13 +1964,10 @@ class Daemon(metaclass=JSONRPCServerType):
         else:
             claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts)
             claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts)
-        return maybe_paginate(
+        return paginate_rows(claims, claim_count, page, page_size)
-            partial(get_records_with_count, claims, claim_count),
-            page, page_size
-        )
 
     @requires(WALLET_COMPONENT)
-    def jsonrpc_claim_search(self, page=None, page_size=None, no_totals=None, **kwargs):
+    async def jsonrpc_claim_search(self, **kwargs):
         """
         Search for stream and channel claims on the blockchain.
 
@@ -2083,18 +2092,18 @@ class Daemon(metaclass=JSONRPCServerType):
 
         Returns: {Paginated[Output]}
         """
-        async def get_claims_with_count(fetch_count, **kwargs):
-            claims, offset, count = await self.ledger.claim_search(**kwargs)
-            return (claims, count)
-
         if kwargs.pop('valid_channel_signature', False):
             kwargs['signature_valid'] = 1
         if kwargs.pop('invalid_channel_signature', False):
             kwargs['signature_valid'] = 0
-        return maybe_paginate(
+        page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
-            get_claims_with_count,
+        kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
-            page, page_size, no_totals, **kwargs
+        txos, offset, total = await self.ledger.claim_search(**kwargs)
-        )
+        result = {"items": txos, "page": page_num, "page_size": page_size}
+        if not kwargs.pop('no_totals', False):
+            result['total_pages'] = int((total + (page_size - 1)) / page_size)
+            result['total_items'] = total
+        return result
 
     CHANNEL_DOC = """
     Create, update, abandon and list your channel claims.
@@ -2460,10 +2469,7 @@ class Daemon(metaclass=JSONRPCServerType):
         else:
             channels = partial(self.ledger.get_channels, wallet=wallet, accounts=wallet.accounts)
             channel_count = partial(self.ledger.get_channel_count, wallet=wallet, accounts=wallet.accounts)
-        return maybe_paginate(
+        return paginate_rows(channels, channel_count, page, page_size)
-            partial(get_records_with_count, channels, channel_count),
-            page, page_size
-        )
 
     @requires(WALLET_COMPONENT)
     async def jsonrpc_channel_export(self, channel_id=None, channel_name=None, account_id=None, wallet_id=None):
@@ -3118,10 +3124,7 @@ class Daemon(metaclass=JSONRPCServerType):
         else:
             streams = partial(self.ledger.get_streams, wallet=wallet, accounts=wallet.accounts)
             stream_count = partial(self.ledger.get_stream_count, wallet=wallet, accounts=wallet.accounts)
-        return maybe_paginate(
+        return paginate_rows(streams, stream_count, page, page_size)
-            partial(get_records_with_count, streams, stream_count),
-            page, page_size
-        )
 
     @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
               DHT_COMPONENT, DATABASE_COMPONENT)
@@ -3224,10 +3227,7 @@ class Daemon(metaclass=JSONRPCServerType):
         else:
             supports = partial(self.ledger.get_supports, wallet=wallet, accounts=wallet.accounts)
             support_count = partial(self.ledger.get_support_count, wallet=wallet, accounts=wallet.accounts)
-        return maybe_paginate(
+        return paginate_rows(supports, support_count, page, page_size)
-            partial(get_records_with_count, supports, support_count),
-            page, page_size
-        )
 
     @requires(WALLET_COMPONENT)
     async def jsonrpc_support_abandon(
@@ -3377,10 +3377,7 @@ class Daemon(metaclass=JSONRPCServerType):
                 self.ledger.get_transaction_history, wallet=wallet, accounts=wallet.accounts)
             transaction_count = partial(
                 self.ledger.get_transaction_history_count, wallet=wallet, accounts=wallet.accounts)
-        return maybe_paginate(
+        return paginate_rows(transactions, transaction_count, page, page_size)
-            partial(get_records_with_count, transactions, transaction_count),
-            page, page_size
-        )
 
     @requires(WALLET_COMPONENT)
     def jsonrpc_transaction_show(self, txid):
@@ -3426,10 +3423,7 @@ class Daemon(metaclass=JSONRPCServerType):
         else:
             utxos = partial(self.ledger.get_utxos, wallet=wallet, accounts=wallet.accounts)
             utxo_count = partial(self.ledger.get_utxo_count, wallet=wallet, accounts=wallet.accounts)
-        return maybe_paginate(
+        return paginate_rows(utxos, utxo_count, page, page_size)
-            partial(get_records_with_count, utxos, utxo_count),
-            page, page_size
-        )
 
     @requires(WALLET_COMPONENT)
     async def jsonrpc_utxo_release(self, account_id=None, wallet_id=None):
@@ -3514,19 +3508,22 @@ class Daemon(metaclass=JSONRPCServerType):
     """
 
     @requires(DHT_COMPONENT)
-    async def jsonrpc_peer_list(self, blob_hash, search_bottom_out_limit=None):
+    async def jsonrpc_peer_list(self, blob_hash, search_bottom_out_limit=None, page=None, page_size=None):
         """
         Get peers for blob hash
 
         Usage:
             peer_list (<blob_hash> | --blob_hash=<blob_hash>)
-            [<search_bottom_out_limit> | --search_bottom_out_limit=<search_bottom_out_limit>]
+                [<search_bottom_out_limit> | --search_bottom_out_limit=<search_bottom_out_limit>]
+                [--page=<page>] [--page_size=<page_size>]
 
         Options:
             --blob_hash=<blob_hash>                                  : (str) find available peers for this blob hash
             --search_bottom_out_limit=<search_bottom_out_limit>      : (int) the number of search probes in a row
                                                                              that don't find any new peers
                                                                              before giving up and returning
+            --page=<page>                                            : (int) page to return during paginating
+            --page_size=<page_size>                                  : (int) number of items on page during pagination
 
         Returns:
             (list) List of contact dictionaries {'address': <peer ip>, 'udp_port': <dht port>, 'tcp_port': <peer port>,
@@ -3555,7 +3552,7 @@ class Daemon(metaclass=JSONRPCServerType):
             }
             for peer in peers
         ]
-        return results
+        return paginate_list(results, page, page_size)
 
     @requires(DATABASE_COMPONENT)
     async def jsonrpc_blob_announce(self, blob_hash=None, stream_hash=None, sd_hash=None):
@@ -3593,7 +3590,7 @@ class Daemon(metaclass=JSONRPCServerType):
 
     @requires(BLOB_COMPONENT, WALLET_COMPONENT)
     async def jsonrpc_blob_list(self, uri=None, stream_hash=None, sd_hash=None, needed=None,
-                                finished=None, page_size=None, page=None):
+                                finished=None, page=None, page_size=None):
         """
         Returns blob hashes. If not given filters, returns all blobs known by the blob manager
 
@@ -3601,8 +3598,7 @@ class Daemon(metaclass=JSONRPCServerType):
             blob_list [--needed] [--finished] [<uri> | --uri=<uri>]
                       [<stream_hash> | --stream_hash=<stream_hash>]
                       [<sd_hash> | --sd_hash=<sd_hash>]
-                      [<page_size> | --page_size=<page_size>]
+                      [--page=<page>] [--page_size=<page_size>]
-                      [<page> | --page=<page>]
 
         Options:
             --needed                     : (bool) only return needed blobs
@@ -3610,8 +3606,8 @@ class Daemon(metaclass=JSONRPCServerType):
             --uri=<uri>                  : (str) filter blobs by stream in a uri
             --stream_hash=<stream_hash>  : (str) filter blobs by stream hash
             --sd_hash=<sd_hash>          : (str) filter blobs by sd hash
-            --page_size=<page_size>      : (int) results page size
+            --page=<page>                : (int) page to return during paginating
-            --page=<page>                : (int) page of results to return
+            --page_size=<page_size>      : (int) number of items on page during pagination
 
         Returns:
             (list) List of blob hashes
@@ -3639,11 +3635,7 @@ class Daemon(metaclass=JSONRPCServerType):
             blobs = [blob_hash for blob_hash in blobs if not self.blob_manager.is_blob_verified(blob_hash)]
         if finished:
             blobs = [blob_hash for blob_hash in blobs if self.blob_manager.is_blob_verified(blob_hash)]
-        page_size = page_size or len(blobs)
+        return paginate_list(blobs, page, page_size)
-        page = page or 0
-        start_index = page * page_size
-        stop_index = start_index + page_size
-        return blobs[start_index:stop_index]
 
     @requires(BLOB_COMPONENT)
     async def jsonrpc_blob_reflect(self, blob_hashes, reflector_server=None):
@@ -4153,9 +4145,3 @@ def get_loggly_query_string(installation_id):
     }
     data = urlencode(params)
     return base_loggly_search_url + data
-
-
-async def get_records_with_count(get_records, get_record_count, fetch_count, **kwargs):
-    records = await get_records(**kwargs)
-    record_count = await get_record_count(**kwargs) if fetch_count else 0
-    return (records, record_count)

lbry/lbry/testcase.py

@@ -276,8 +276,11 @@ class CommandTestCase(IntegrationTestCase):
         return await self.out(self.daemon.jsonrpc_resolve(uri))
 
     async def claim_search(self, **kwargs):
-        return await self.out(self.daemon.jsonrpc_claim_search(**kwargs))
+        return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
 
     @staticmethod
     def get_claim_id(tx):
         return tx['outputs'][0]['claim_id']
+
+    def assertItemCount(self, result, count):
+        self.assertEqual(result['total_items'], count)

lbry/tests/integration/test_account_commands.py

@@ -13,55 +13,55 @@ def extract(d, keys):
 class AccountManagement(CommandTestCase):
     async def test_account_list_set_create_remove_add(self):
         # check initial account
-        response = await self.daemon.jsonrpc_account_list()
+        accounts = await self.daemon.jsonrpc_account_list()
-        self.assertEqual(len(response['lbc_regtest']), 1)
+        self.assertItemCount(accounts, 1)
 
         # change account name and gap
-        account_id = response['lbc_regtest'][0]['id']
+        account_id = accounts['items'][0]['id']
         self.daemon.jsonrpc_account_set(
             account_id=account_id, new_name='test account',
             receiving_gap=95, receiving_max_uses=96,
             change_gap=97, change_max_uses=98
         )
-        response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
+        accounts = (await self.daemon.jsonrpc_account_list())['items'][0]
-        self.assertEqual(response['name'], 'test account')
+        self.assertEqual(accounts['name'], 'test account')
         self.assertEqual(
-            response['address_generator']['receiving'],
+            accounts['address_generator']['receiving'],
             {'gap': 95, 'maximum_uses_per_address': 96}
         )
         self.assertEqual(
-            response['address_generator']['change'],
+            accounts['address_generator']['change'],
             {'gap': 97, 'maximum_uses_per_address': 98}
         )
 
         # create another account
         await self.daemon.jsonrpc_account_create('second account')
-        response = await self.daemon.jsonrpc_account_list()
+        accounts = await self.daemon.jsonrpc_account_list()
-        self.assertEqual(len(response['lbc_regtest']), 2)
+        self.assertItemCount(accounts, 2)
-        self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
+        self.assertEqual(accounts['items'][1]['name'], 'second account')
-        account_id2 = response['lbc_regtest'][1]['id']
+        account_id2 = accounts['items'][1]['id']
 
         # make new account the default
         self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
-        response = await self.daemon.jsonrpc_account_list(show_seed=True)
+        accounts = await self.daemon.jsonrpc_account_list(show_seed=True)
-        self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
+        self.assertEqual(accounts['items'][0]['name'], 'second account')
 
-        account_seed = response['lbc_regtest'][1]['seed']
+        account_seed = accounts['items'][1]['seed']
 
         # remove account
-        self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
+        self.daemon.jsonrpc_account_remove(accounts['items'][1]['id'])
-        response = await self.daemon.jsonrpc_account_list()
+        accounts = await self.daemon.jsonrpc_account_list()
-        self.assertEqual(len(response['lbc_regtest']), 1)
+        self.assertItemCount(accounts, 1)
 
         # add account
         await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
-        response = await self.daemon.jsonrpc_account_list()
+        accounts = await self.daemon.jsonrpc_account_list()
-        self.assertEqual(len(response['lbc_regtest']), 2)
+        self.assertItemCount(accounts, 2)
-        self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
+        self.assertEqual(accounts['items'][1]['name'], 'recreated account')
 
         # list specific account
-        response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
+        accounts = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
-        self.assertEqual(response['name'], 'recreated account')
+        self.assertEqual(accounts['items'][0]['name'], 'recreated account')
 
     async def test_wallet_migration(self):
         # null certificates should get deleted
@@ -75,10 +75,10 @@ class AccountManagement(CommandTestCase):
         self.assertEqual(list(self.account.channel_keys.keys()), [keys[2]])
 
     async def assertFindsClaims(self, claim_names, awaitable):
-        self.assertEqual(claim_names, [txo.claim_name for txo in await awaitable])
+        self.assertEqual(claim_names, [txo.claim_name for txo in (await awaitable)['items']])
 
     async def assertOutputAmount(self, amounts, awaitable):
-        self.assertEqual(amounts, [dewies_to_lbc(txo.amount) for txo in await awaitable])
+        self.assertEqual(amounts, [dewies_to_lbc(txo.amount) for txo in (await awaitable)['items']])
 
     async def test_commands_across_accounts(self):
         channel_list = self.daemon.jsonrpc_channel_list
@@ -131,12 +131,13 @@ class AccountManagement(CommandTestCase):
         support2 = await self.support_create(
             self.get_claim_id(stream2), '0.01', account_id=second_account.id, funding_account_ids=[default_account.id]
         )
-        self.assertEqual([support2['txid'], support1['txid']], [txo.tx_ref.id for txo in await support_list()])
+        self.assertEqual([support2['txid'], support1['txid']], [txo.tx_ref.id for txo in (await support_list())['items']])
-        self.assertEqual([support1['txid']], [txo.tx_ref.id for txo in await support_list(account_id=default_account.id)])
+        self.assertEqual([support1['txid']], [txo.tx_ref.id for txo in (await support_list(account_id=default_account.id))['items']])
-        self.assertEqual([support2['txid']], [txo.tx_ref.id for txo in await support_list(account_id=second_account.id)])
+        self.assertEqual([support2['txid']], [txo.tx_ref.id for txo in (await support_list(account_id=second_account.id))['items']])
 
         history = await self.daemon.jsonrpc_transaction_list()
-        self.assertEqual(len(history), 8)
+        self.assertItemCount(history, 8)
+        history = history['items']
         self.assertEqual(extract(history[0]['support_info'][0], ['claim_name', 'is_tip', 'amount', 'balance_delta']), {
             'claim_name': 'stream-in-account2',
             'is_tip': False,

lbry/tests/integration/test_chris45.py

@@ -23,8 +23,8 @@ class EpicAdventuresOfChris45(CommandTestCase):
 
         # Do we have it locally?
         channels = await self.out(self.daemon.jsonrpc_channel_list())
-        self.assertEqual(len(channels), 1)
+        self.assertItemCount(channels, 1)
-        self.assertEqual(channels[0]['name'], '@spam')
+        self.assertEqual(channels['items'][0]['name'], '@spam')
 
         # As the new channel claim travels through the intertubes and makes its
         # way into the mempool and then a block and then into the claimtrie,

lbry/tests/integration/test_claim_commands.py

@@ -43,20 +43,20 @@ class ClaimSearchCommand(ClaimTestCase):
         self.channel = await self.channel_create('@abc', '1.0')
         self.channel_id = self.get_claim_id(self.channel)
 
-    async def create_lots_of_streams(self, claims=4, blocks=3):
+    async def create_lots_of_streams(self):
         tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True)
         await self.confirm_tx(tx.id)
-        # 4 claims per block, 3 blocks (by default). Sorted by height (descending) then claim name (ascending).
+        # 4 claims per block, 3 blocks. Sorted by height (descending) then claim name (ascending).
         self.streams = []
-        for j in range(blocks):
+        for j in range(4):
             same_height_claims = []
-            for k in range(claims - 1):
+            for k in range(5):
                 claim_tx = await self.stream_create(
                     f'c{j}-{k}', '0.000001', channel_id=self.channel_id, confirm=False)
                 same_height_claims.append(claim_tx['outputs'][0]['name'])
                 await self.on_transaction_dict(claim_tx)
             claim_tx = await self.stream_create(
-                f'c{j}-{claims - 1}', '0.000001', channel_id=self.channel_id, confirm=True)
+                f'c{j}-6', '0.000001', channel_id=self.channel_id, confirm=True)
             same_height_claims.append(claim_tx['outputs'][0]['name'])
             self.streams = same_height_claims + self.streams
 
@@ -139,40 +139,44 @@ class ClaimSearchCommand(ClaimTestCase):
 
     async def test_pagination(self):
         await self.create_channel()
-        await self.create_lots_of_streams(10, 10)
+        await self.create_lots_of_streams()
 
-        page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
+        # with and without totals
-        page_claim_ids = [item['name'] for item in page['items']]
+        results = await self.daemon.jsonrpc_claim_search()
-        self.assertEqual(page_claim_ids, self.streams[:20])
+        self.assertEqual(results['total_pages'], 2)
-
+        self.assertEqual(results['total_items'], 25)
-        page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
+        results = await self.daemon.jsonrpc_claim_search(no_totals=True)
-        page_claim_ids = [item['name'] for item in page['items']]
-        self.assertEqual(page_claim_ids, self.streams[:6])
-
-        page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
-        page_claim_ids = [item['name'] for item in page['items']]
-        self.assertEqual(page_claim_ids, self.streams[6:(2 * 6)])
-
-        page = await self.claim_search(page=1, channel='@abc', order_by=['height', '^name'])
-        page_claim_ids = [item['name'] for item in page['items']]
-        self.assertEqual(page_claim_ids, self.streams[:DEFAULT_PAGE_SIZE])
-
-        page = await self.claim_search(page=2, channel='@abc', order_by=['height', '^name'])
-        page_claim_ids = [item['name'] for item in page['items']]
-        self.assertEqual(page_claim_ids, self.streams[DEFAULT_PAGE_SIZE:(2 * DEFAULT_PAGE_SIZE)])
-
-        out_of_bounds = await self.claim_search(page=20, page_size=20, channel='@abc')
-        self.assertEqual(out_of_bounds['items'], [])
-
-        total_claims = 10 * 10 + 1
-        results = await self.claim_search(page=1)
-        self.assertEqual(results['total_pages'], (total_claims + DEFAULT_PAGE_SIZE - 1) // DEFAULT_PAGE_SIZE)
-        self.assertEqual(results['total_items'], total_claims)
-
-        results = await self.claim_search(page=1, no_totals=True)
         self.assertNotIn('total_pages', results)
         self.assertNotIn('total_items', results)
 
+        # defaults
+        page = await self.claim_search(channel='@abc', order_by=['height', '^name'])
+        page_claim_ids = [item['name'] for item in page]
+        self.assertEqual(page_claim_ids, self.streams[:DEFAULT_PAGE_SIZE])
+
+        # page with default page_size
+        page = await self.claim_search(page=2, channel='@abc', order_by=['height', '^name'])
+        page_claim_ids = [item['name'] for item in page]
+        self.assertEqual(page_claim_ids, self.streams[DEFAULT_PAGE_SIZE:(DEFAULT_PAGE_SIZE*2)])
+
+        # page_size larger than dataset
+        page = await self.claim_search(page_size=50, channel='@abc', order_by=['height', '^name'])
+        page_claim_ids = [item['name'] for item in page]
+        self.assertEqual(page_claim_ids, self.streams)
+
+        # page_size less than dataset
+        page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
+        page_claim_ids = [item['name'] for item in page]
+        self.assertEqual(page_claim_ids, self.streams[:6])
+
+        # page and page_size
+        page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
+        page_claim_ids = [item['name'] for item in page]
+        self.assertEqual(page_claim_ids, self.streams[6:12])
+
+        out_of_bounds = await self.claim_search(page=4, page_size=20, channel='@abc')
+        self.assertEqual(out_of_bounds, [])
+
     async def test_tag_search(self):
         claim1 = await self.stream_create('claim1', tags=['aBc'])
         claim2 = await self.stream_create('claim2', tags=['#abc', 'def'])
@@ -330,7 +334,7 @@ class ChannelCommands(CommandTestCase):
     async def test_create_channel_names(self):
         # claim new name
         await self.channel_create('@foo')
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1)
         await self.assertBalance(self.account, '8.991893')
 
         # fail to claim duplicate
@@ -342,19 +346,19 @@ class ChannelCommands(CommandTestCase):
             await self.channel_create('foo')
 
         # nothing's changed after failed attempts
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1)
         await self.assertBalance(self.account, '8.991893')
 
         # succeed overriding duplicate restriction
         await self.channel_create('@foo', allow_duplicate_name=True)
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 2)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 2)
         await self.assertBalance(self.account, '7.983786')
 
     async def test_channel_bids(self):
         # enough funds
         tx = await self.channel_create('@foo', '5.0')
         claim_id = self.get_claim_id(tx)
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1)
         await self.assertBalance(self.account, '4.991893')
 
         # bid preserved on update
@@ -371,14 +375,14 @@ class ChannelCommands(CommandTestCase):
         with self.assertRaisesRegex(
                 InsufficientFundsError, "Not enough funds to cover this transaction."):
             await self.channel_create('@foo2', '9.0')
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1)
         await self.assertBalance(self.account, '5.991447')
 
         # spend exactly amount available, no change
         tx = await self.channel_create('@foo3', '5.981266')
         await self.assertBalance(self.account, '0.0')
         self.assertEqual(len(tx['outputs']), 1)  # no change
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 2)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 2)
 
     async def test_setting_channel_fields(self):
         values = {
@@ -455,16 +459,16 @@ class ChannelCommands(CommandTestCase):
         account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
 
         # before moving
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 3)
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 0)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 0)
 
         other_address = await account2.receiving.get_or_create_usable_address()
         tx = await self.out(self.channel_update(claim_id, claim_address=other_address))
 
         # after moving
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 3)
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=self.account.id)), 2)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=self.account.id), 2)
-        self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 1)
 
     async def test_channel_export_import_before_sending_channel(self):
         # export
@@ -475,9 +479,9 @@ class ChannelCommands(CommandTestCase):
 
         # import
         daemon2 = await self.add_daemon()
-        self.assertEqual(0, len(await daemon2.jsonrpc_channel_list()))
+        self.assertItemCount(await daemon2.jsonrpc_channel_list(), 0)
         await daemon2.jsonrpc_channel_import(exported_data)
-        channels = await daemon2.jsonrpc_channel_list()
+        channels = (await daemon2.jsonrpc_channel_list())['items']
         self.assertEqual(1, len(channels))
         self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
 
@@ -499,11 +503,11 @@ class ChannelCommands(CommandTestCase):
             await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=self.account.id)
         # channel is in account2
         await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=account2.id)
-        result = await self.out(self.daemon.jsonrpc_channel_list())
+        result = (await self.out(self.daemon.jsonrpc_channel_list()))['items']
         self.assertEqual(result[0]['amount'], '2.0')
         # check all accounts for channel
         await self.channel_update(self.get_claim_id(channel), bid='3.0')
-        result = await self.out(self.daemon.jsonrpc_channel_list())
+        result = (await self.out(self.daemon.jsonrpc_channel_list()))['items']
         self.assertEqual(result[0]['amount'], '3.0')
         await self.channel_abandon(self.get_claim_id(channel))
 
@@ -527,7 +531,7 @@ class StreamCommands(ClaimTestCase):
     async def test_create_stream_names(self):
         # claim new name
         await self.stream_create('foo')
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1)
         await self.assertBalance(self.account, '8.993893')
 
         # fail to claim duplicate
@@ -540,19 +544,19 @@ class StreamCommands(ClaimTestCase):
                 Exception, "Stream names cannot start with '@' symbol."):
             await self.stream_create('@foo')
 
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1)
         await self.assertBalance(self.account, '8.993893')
 
         # succeed overriding duplicate restriction
         await self.stream_create('foo', allow_duplicate_name=True)
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 2)
         await self.assertBalance(self.account, '7.987786')
 
     async def test_stream_bids(self):
         # enough funds
         tx = await self.stream_create('foo', '2.0')
         claim_id = self.get_claim_id(tx)
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1)
         await self.assertBalance(self.account, '7.993893')
 
         # bid preserved on update
@@ -569,14 +573,14 @@ class StreamCommands(ClaimTestCase):
         with self.assertRaisesRegex(
                 InsufficientFundsError, "Not enough funds to cover this transaction."):
             await self.stream_create('foo2', '9.0')
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1)
         await self.assertBalance(self.account, '6.993319')
 
         # spend exactly amount available, no change
         tx = await self.stream_create('foo3', '6.98523')
         await self.assertBalance(self.account, '0.0')
         self.assertEqual(len(tx['outputs']), 1)  # no change
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 2)
 
     async def test_stream_update_and_abandon_across_accounts(self):
         account2 = await self.daemon.jsonrpc_account_create('second account')
@@ -586,11 +590,11 @@ class StreamCommands(ClaimTestCase):
             await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=self.account.id)
         # stream is in account2
         await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=account2.id)
-        result = await self.out(self.daemon.jsonrpc_stream_list())
+        result = (await self.out(self.daemon.jsonrpc_stream_list()))['items']
         self.assertEqual(result[0]['amount'], '2.0')
         # check all accounts for stream
         await self.stream_update(self.get_claim_id(stream), bid='3.0')
-        result = await self.out(self.daemon.jsonrpc_stream_list())
+        result = (await self.out(self.daemon.jsonrpc_stream_list()))['items']
         self.assertEqual(result[0]['amount'], '3.0')
         await self.stream_abandon(self.get_claim_id(stream))
 
@@ -614,18 +618,18 @@ class StreamCommands(ClaimTestCase):
         baz_id = self.get_claim_id(baz_tx)
 
         channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id))
-        self.assertEqual(len(channels), 1)
+        self.assertItemCount(channels, 1)
-        self.assertEqual(channels[0]['name'], '@spam')
+        self.assertEqual(channels['items'][0]['name'], '@spam')
         self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list(account1_id)))
 
         channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id))
-        self.assertEqual(len(channels), 1)
+        self.assertItemCount(channels, 1)
-        self.assertEqual(channels[0]['name'], '@baz')
+        self.assertEqual(channels['items'][0]['name'], '@baz')
 
         channels = await self.out(self.daemon.jsonrpc_channel_list())
-        self.assertEqual(len(channels), 2)
+        self.assertItemCount(channels, 2)
-        self.assertEqual(channels[0]['name'], '@baz')
+        self.assertEqual(channels['items'][0]['name'], '@baz')
-        self.assertEqual(channels[1]['name'], '@spam')
+        self.assertEqual(channels['items'][1]['name'], '@spam')
 
         # defaults to using all accounts to lookup channel
         await self.stream_create('hovercraft1', '0.1', channel_id=baz_id)
@@ -817,17 +821,17 @@ class StreamCommands(ClaimTestCase):
         account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
 
         # before sending
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 4)
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=self.account.id)), 4)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=self.account.id), 4)
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 0)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=account2_id), 0)
 
         other_address = await account2.receiving.get_or_create_usable_address()
         tx = await self.out(self.stream_update(claim_id, claim_address=other_address))
 
         # after sending
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 4)
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=self.account.id)), 3)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=self.account.id), 3)
-        self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1)
+        self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=account2_id), 1)
 
     async def test_setting_fee_fields(self):
         tx = await self.out(self.stream_create('paid-stream'))
@@ -1043,7 +1047,7 @@ class StreamCommands(ClaimTestCase):
 
         tx = await self.stream_create(bid='2.5')  # creates new claim
         claim_id = self.get_claim_id(tx)
-        txs = await self.out(self.daemon.jsonrpc_transaction_list())
+        txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
         self.assertEqual(len(txs[0]['claim_info']), 1)
         self.assertEqual(txs[0]['confirmations'], 1)
         self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
@@ -1057,7 +1061,7 @@ class StreamCommands(ClaimTestCase):
         self.assertEqual(0, len(self.daemon.jsonrpc_file_list()))
 
         await self.stream_update(claim_id, bid='1.0')  # updates previous claim
-        txs = await self.out(self.daemon.jsonrpc_transaction_list())
+        txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
         self.assertEqual(len(txs[0]['update_info']), 1)
         self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
         self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id)
@@ -1066,7 +1070,7 @@ class StreamCommands(ClaimTestCase):
         await self.assertBalance(self.account, '8.9796765')
 
         await self.stream_abandon(claim_id)
-        txs = await self.out(self.daemon.jsonrpc_transaction_list())
+        txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
         self.assertEqual(len(txs[0]['abandon_info']), 1)
         self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
         self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id)
@@ -1169,7 +1173,7 @@ class SupportCommands(CommandTestCase):
         await self.assertBalance(account2,     '3.9998585')
 
         # verify that the incoming tip is marked correctly as is_tip=True in account1
-        txs = await self.out(self.daemon.jsonrpc_transaction_list(self.account.id))
+        txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items']
         self.assertEqual(len(txs[0]['support_info']), 1)
         self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
         self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id)
@@ -1178,9 +1182,9 @@ class SupportCommands(CommandTestCase):
         self.assertEqual(txs[0]['fee'], '0.0')
 
         # verify that the outgoing tip is marked correctly as is_tip=True in account2
-        txs2 = await self.out(
+        txs2 = (await self.out(
             self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
-        )
+        ))['items']
         self.assertEqual(len(txs2[0]['support_info']), 1)
         self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
         self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
@@ -1200,7 +1204,7 @@ class SupportCommands(CommandTestCase):
         await self.assertBalance(account2,     '1.999717')
 
         # verify that the outgoing support is marked correctly as is_tip=False in account2
-        txs2 = await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2'))
+        txs2 = (await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')))['items']
         self.assertEqual(len(txs2[0]['support_info']), 1)
         self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
         self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)

lbry/tests/integration/test_file_commands.py

@@ -263,7 +263,7 @@ class FileCommands(CommandTestCase):
         BlobDownloader.BAN_FACTOR = .5  # fixme: temporary field, will move to connection manager or a conf
         tx = await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
         sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
-        missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2]
+        missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))['items'][-2]
         await self.daemon.jsonrpc_file_delete(claim_name='foo')
         # backup blob
         missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)

lbry/tests/integration/test_other_commands.py

@@ -5,8 +5,8 @@ class AddressManagement(CommandTestCase):
 
     async def test_address_list(self):
         addresses = await self.out(self.daemon.jsonrpc_address_list())
-        self.assertEqual(27, len(addresses))
+        self.assertItemCount(addresses, 27)
 
-        single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address']))
+        single = await self.out(self.daemon.jsonrpc_address_list(addresses['items'][11]['address']))
-        self.assertEqual(1, len(single))
+        self.assertItemCount(single, 1)
-        self.assertEqual(single[0], addresses[11])
+        self.assertEqual(single['items'][0], addresses['items'][11])

lbry/tests/integration/test_wallet_commands.py

@@ -49,12 +49,12 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
             "two": "2", "conflict": "2", "another": "A"
         })
 
-        self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 1)
+        self.assertItemCount(await daemon.jsonrpc_account_list(), 1)
 
         data = await daemon2.jsonrpc_sync_apply('password')
         await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
 
-        self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 2)
+        self.assertItemCount(await daemon.jsonrpc_account_list(), 2)
         self.assertDictEqual(
             # "two" key added and "conflict" value changed to "2"
             daemon.jsonrpc_preference_get(),
@@ -66,9 +66,9 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
         await self.confirm_tx(channel.id, self.daemon2.ledger)
 
         # both daemons will have the channel but only one has the cert so far
-        self.assertEqual(len(await daemon.jsonrpc_channel_list()), 1)
+        self.assertItemCount(await daemon.jsonrpc_channel_list(), 1)
         self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0)
-        self.assertEqual(len(await daemon2.jsonrpc_channel_list()), 1)
+        self.assertItemCount(await daemon2.jsonrpc_channel_list(), 1)
         self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1)
 
         data = await daemon2.jsonrpc_sync_apply('password')

torba/torba/client/baseaccount.py

@@ -334,6 +334,7 @@ class BaseAccount:
         details = {
             'id': self.id,
             'name': self.name,
+            'ledger': self.ledger.get_id(),
             'coins': round(satoshis/COIN, 2),
             'satoshis': satoshis,
             'encrypted': self.encrypted,

torba/torba/client/basedatabase.py

@@ -612,7 +612,7 @@ class BaseDatabase(SQLiteMixin):
             )
         return addresses
 
-    async def get_address_count(self, **constraints):
+    async def get_address_count(self, cols=None, **constraints):
         count = await self.select_addresses('count(*)', **constraints)
         return count[0][0]
 

torba/torba/client/wallet.py

@@ -109,14 +109,12 @@ class Wallet:
         ] if account_ids else self.accounts
 
     async def get_detailed_accounts(self, **kwargs):
-        ledgers = {}
+        accounts = []
         for i, account in enumerate(self.accounts):
             details = await account.get_details(**kwargs)
             details['is_default'] = i == 0
-            ledger_id = account.ledger.get_id()
+            accounts.append(details)
-            ledgers.setdefault(ledger_id, [])
+        return accounts
-            ledgers[ledger_id].append(details)
-        return ledgers
 
     @classmethod
     def from_storage(cls, storage: 'WalletStorage', manager: 'basemanager.BaseWalletManager') -> 'Wallet':