fixing stalls in tests

lbry/testcase.py

@@ -265,6 +265,13 @@ class IntegrationTestCase(AsyncioTestCase):
     def broadcast(self, tx):
         return self.ledger.broadcast(tx)
 
+    async def broadcast_and_confirm(self, tx, ledger=None):
+        ledger = ledger or self.ledger
+        notifications = asyncio.create_task(ledger.wait(tx))
+        await ledger.broadcast(tx)
+        await notifications
+        await self.generate_and_wait(1, [tx.id], ledger)
+
     async def on_header(self, height):
         if self.ledger.headers.height < height:
             await self.ledger.on_header.where(
@@ -277,7 +284,7 @@ class IntegrationTestCase(AsyncioTestCase):
         txid = None
         done = False
         watcher = (ledger or self.ledger).on_transaction.where(
-            lambda e: e.tx.id == txid or tx_watch.append(e.tx.id) or done
+            lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id)
         )
 
         txid = await self.blockchain.send_to_address(address, amount)
@@ -290,7 +297,7 @@ class IntegrationTestCase(AsyncioTestCase):
     async def generate_and_wait(self, blocks_to_generate, txids, ledger=None):
         if blocks_to_generate > 0:
             watcher = (ledger or self.ledger).on_transaction.where(
-                lambda e: (e.tx.id in txids and txids.remove(e.tx.id)) or len(txids) <= 0  # relies on remove returning None
+                lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1]  # multi-statement lambda
             )
             await self.blockchain.generate(blocks_to_generate)
             await watcher

lbry/wallet/ledger.py

@@ -365,6 +365,10 @@ class Ledger(metaclass=LedgerRegistry):
         await self.db.close()
         await self.headers.close()
 
+    async def tasks_are_done(self):
+        await self._update_tasks.done.wait()
+        await self._other_tasks.done.wait()
+
     @property
     def local_height_including_downloaded_height(self):
         return max(self.headers.height, self._download_height)

tests/integration/blockchain/test_account_commands.py

@@ -103,7 +103,7 @@ class AccountManagement(CommandTestCase):
         second_account = await self.daemon.jsonrpc_account_create('second account')
 
         tx = await self.daemon.jsonrpc_account_send(
-            '0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id)
+            '0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id), blocking=True
         )
         await self.confirm_tx(tx.id)
         await self.assertOutputAmount(['0.05', '9.949876'], utxo_list())

tests/integration/blockchain/test_wallet_commands.py

@@ -73,7 +73,8 @@ class WalletCommands(CommandTestCase):
     async def test_balance_caching(self):
         account2 = await self.daemon.jsonrpc_account_create("Tip-er")
         address2 = await self.daemon.jsonrpc_address_unused(account2.id)
-        await self.send_to_address_and_wait(address2, 10, 1)
+        await self.send_to_address_and_wait(address2, 10, 2)
+        await self.ledger.tasks_are_done()  # don't mess with the query count while we need it
 
         wallet_balance = self.daemon.jsonrpc_wallet_balance
         ledger = self.ledger
@@ -88,14 +89,16 @@ class WalletCommands(CommandTestCase):
         self.assertIsNone(ledger._balance_cache.get(self.account.id))
 
         query_count += 2
-        self.assertEqual(await wallet_balance(), expected)
+        balance = await wallet_balance()
         self.assertEqual(self.ledger.db.db.query_count, query_count)
+        self.assertEqual(balance, expected)
         self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0')
         self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0')
 
         # calling again uses cache
-        self.assertEqual(await wallet_balance(), expected)
+        balance = await wallet_balance()
         self.assertEqual(self.ledger.db.db.query_count, query_count)
+        self.assertEqual(balance, expected)
         self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0')
         self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0')
 
@@ -151,7 +154,7 @@ class WalletCommands(CommandTestCase):
         address2 = await self.daemon.jsonrpc_address_unused(account2.id)
 
         # send lbc to someone else
-        tx = await self.daemon.jsonrpc_account_send('1.0', address2)
+        tx = await self.daemon.jsonrpc_account_send('1.0', address2, blocking=True)
         await self.confirm_tx(tx.id)
         self.assertEqual(await account_balance(), {
             'total': '8.97741',
@@ -184,7 +187,7 @@ class WalletCommands(CommandTestCase):
         })
 
         # tip claimed
-        tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0)
+        tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0, blocking=True)
         await self.confirm_tx(tx.id)
         self.assertEqual(await account_balance(), {
             'total': '9.277303',
@@ -290,7 +293,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
             '3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1'
             '66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16'
             'a97a6d6a4a8effd29d748901bb9789352519cd00b13d'
-        ), self.daemon2)
+        ), self.daemon2, blocking=True)
         await self.confirm_tx(channel['txid'], self.daemon2.ledger)
 
         # both daemons will have the channel but only one has the cert so far

tests/integration/claims/test_claim_commands.py

@@ -494,8 +494,7 @@ class ClaimSearchCommand(ClaimTestCase):
         tx = await Transaction.claim_create(
             'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account)
         await tx.sign([self.account])
-        await self.broadcast(tx)
+        await self.broadcast_and_confirm(tx)
-        await self.confirm_tx(tx.id)
 
         octet = await self.stream_create()
         video = await self.stream_create('chrome', file_path=self.video_file_name)
@@ -1226,7 +1225,7 @@ class ChannelCommands(CommandTestCase):
         data_to_sign = "CAFEBABE"
         # claim new name
         await self.channel_create('@someotherchan')
-        channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1')
+        channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1', blocking=True)
         await self.confirm_tx(channel_tx.id)
         channel = channel_tx.outputs[0]
         signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
@@ -1373,7 +1372,7 @@ class StreamCommands(ClaimTestCase):
         self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available'])
 
         result = await self.out(self.daemon.jsonrpc_account_send(
-            '5.0', await self.daemon.jsonrpc_address_unused(account2_id)
+            '5.0', await self.daemon.jsonrpc_address_unused(account2_id), blocking=True
         ))
         await self.confirm_tx(result['txid'])
 
@@ -2177,7 +2176,7 @@ class SupportCommands(CommandTestCase):
         tip = await self.out(
             self.daemon.jsonrpc_support_create(
                 claim_id, '1.0', True, account_id=account2.id, wallet_id='wallet2',
-                funding_account_ids=[account2.id])
+                funding_account_ids=[account2.id], blocking=True)
         )
         await self.confirm_tx(tip['txid'])
 
@@ -2209,7 +2208,7 @@ class SupportCommands(CommandTestCase):
         support = await self.out(
             self.daemon.jsonrpc_support_create(
                 claim_id, '2.0', False, account_id=account2.id, wallet_id='wallet2',
-                funding_account_ids=[account2.id])
+                funding_account_ids=[account2.id], blocking=True)
         )
         await self.confirm_tx(support['txid'])
 

tests/integration/datanetwork/test_file_commands.py

@@ -37,8 +37,7 @@ class FileCommands(CommandTestCase):
                 tx_to_update.outputs[0], claim, 1, address, [self.account], self.account
             )
         await tx.sign([self.account])
-        await self.broadcast(tx)
+        await self.broadcast_and_confirm(tx)
-        await self.confirm_tx(tx.id)
         self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
         self.client_session._session.add_dht_node(('localhost', 4040))
         self.client_session.wait_start = False  # fixme: this is super slow on tests
@@ -506,8 +505,7 @@ class FileCommands(CommandTestCase):
         tx.outputs[0].claim.stream.fee.address_bytes = b''
         tx.outputs[0].script.generate()
         await tx.sign([self.account])
-        await self.broadcast(tx)
+        await self.broadcast_and_confirm(tx)
-        await self.confirm_tx(tx.id)
 
     async def __raw_value_update_no_fee_amount(self, tx, claim_address):
         tx = await self.daemon.jsonrpc_stream_update(
@@ -517,8 +515,7 @@ class FileCommands(CommandTestCase):
         tx.outputs[0].claim.stream.fee.message.ClearField('amount')
         tx.outputs[0].script.generate()
         await tx.sign([self.account])
-        await self.broadcast(tx)
+        await self.broadcast_and_confirm(tx)
-        await self.confirm_tx(tx.id)
 
 
 class DiskSpaceManagement(CommandTestCase):

tests/integration/other/test_chris45.py

@@ -80,7 +80,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
 
         # After some soul searching Chris decides that his story needs more
         # heart and a better ending. He takes down the story and begins the rewrite.
-        abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=False))
+        abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=True))
         self.assertEqual(abandon['inputs'][0]['claim_id'], claim_id)
         await self.confirm_tx(abandon['txid'])
 
@@ -103,7 +103,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
         # 1 LBC to which Chris readily obliges
         ramsey_account_id = (await self.out(self.daemon.jsonrpc_account_create("Ramsey")))['id']
         ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
-        result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address))
+        result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address, blocking=True))
         self.assertIn("txid", result)
         await self.confirm_tx(result['txid'])
 
@@ -133,7 +133,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
         # And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
         # Now to keep this claim winning on the lbry blockchain he immediately supports the claim
         tx = await self.out(self.daemon.jsonrpc_support_create(
-            claim_id2, '0.2', account_id=ramsey_account_id
+            claim_id2, '0.2', account_id=ramsey_account_id, blocking=True
         ))
         await self.confirm_tx(tx['txid'])
 
@@ -147,7 +147,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
         # Now he also wanted to support the original creator of the Award Winning Novel
         # So he quickly decides to send a tip to him
         tx = await self.out(
-            self.daemon.jsonrpc_support_create(claim_id2, '0.3', tip=True, account_id=ramsey_account_id)
+            self.daemon.jsonrpc_support_create(claim_id2, '0.3', tip=True, account_id=ramsey_account_id, blocking=True)
         )
         await self.confirm_tx(tx['txid'])
 
@@ -158,7 +158,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
         await self.generate(5)
 
         # Seeing the ravishing success of his novel Chris adds support to his claim too
-        tx = await self.out(self.daemon.jsonrpc_support_create(claim_id2, '0.4'))
+        tx = await self.out(self.daemon.jsonrpc_support_create(claim_id2, '0.4', blocking=True))
         await self.confirm_tx(tx['txid'])
 
         # And check if his support showed up
@@ -183,7 +183,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
 
         # But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
         # Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
-        abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=False))
+        abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=True))
         self.assertTrue(abandon['inputs'][0]['txid'], tx['txid'])
         await self.confirm_tx(abandon['txid'])
 

tests/integration/takeovers/test_resolve_command.py

@@ -1,6 +1,7 @@
 import asyncio
 import json
 import hashlib
+import sys
 from bisect import bisect_right
 from binascii import hexlify, unhexlify
 from collections import defaultdict
@@ -91,7 +92,7 @@ class BaseResolveTestCase(CommandTestCase):
         self.assertEqual(len(claim_from_es[0]), 1)
         self.assertMatchESClaim(claim_from_es[0][0], claim)
         self._check_supports(claim.claim_hash.hex(), expected.get('supports', []),
-                             claim_from_es[0][0]['support_amount'], expected['effectiveamount'] > 0)
+                             claim_from_es[0][0]['support_amount'])
 
     async def assertMatchClaim(self, name, claim_id, is_active_in_lbrycrd=True):
         claim = await self.conductor.spv_node.server.bp.db.fs_getclaimbyid(claim_id)
@@ -110,7 +111,7 @@ class BaseResolveTestCase(CommandTestCase):
             expected['claims'][0]['lasttakeoverheight'] = expected['lasttakeoverheight']
             self.assertMatchDBClaim(expected['claims'][0], claim)
             self._check_supports(claim.claim_hash.hex(), expected['claims'][0].get('supports', []),
-                                 claim_from_es[0][0]['support_amount'], is_active_in_lbrycrd)
+                                 claim_from_es[0][0]['support_amount'])
         else:
             if 'claims' in expected and expected['claims'] is not None:
                 # ensure that if we do have the matching claim that it is not active
@@ -121,19 +122,30 @@ class BaseResolveTestCase(CommandTestCase):
         self.assertEqual(claim_id, (await self.assertMatchWinningClaim(name)).claim_hash.hex())
         await self.assertMatchClaimsForName(name)
 
-    def _check_supports(self, claim_id, lbrycrd_supports, es_support_amount, is_active_in_lbrycrd=True):
+    def _check_supports(self, claim_id, lbrycrd_supports, es_support_amount):
-        total_amount = 0
+        total_lbrycrd_amount = 0.0
-        db = self.conductor.spv_node.server.db
+        total_es_amount = 0.0
+        active_es_amount = 0.0
+        db = self.conductor.spv_node.server.bp.db
+        es_supports = db.get_supports(bytes.fromhex(claim_id))
 
-        for i, (tx_num, position, amount) in enumerate(db.get_supports(bytes.fromhex(claim_id))):
+        # we're only concerned about active supports here, and they should match
-            total_amount += amount
+        self.assertTrue(len(es_supports) >= len(lbrycrd_supports))
-            if is_active_in_lbrycrd:
+
-                support = lbrycrd_supports[i]
+        for i, (tx_num, position, amount) in enumerate(es_supports):
-                self.assertEqual(support['txid'], db.prefix_db.tx_hash.get(tx_num, deserialize_value=False)[::-1].hex())
+            total_es_amount += amount
-                self.assertEqual(support['n'], position)
+            valid_height = db.get_activation(tx_num, position, is_support=True)
-                self.assertEqual(support['height'], bisect_right(db.tx_counts, tx_num))
+            if valid_height > db.db_height:
-                self.assertEqual(support['validatheight'], db.get_activation(tx_num, position, is_support=True))
+                continue
-        self.assertEqual(total_amount, es_support_amount, f"lbrycrd support amount: {total_amount} vs es: {es_support_amount}")
+            active_es_amount += amount
+            txid = db.prefix_db.tx_hash.get(tx_num, deserialize_value=False)[::-1].hex()
+            support = next(filter(lambda s: s['txid'] == txid and s['n'] == position, lbrycrd_supports))
+            total_lbrycrd_amount += support['amount']
+            self.assertEqual(support['height'], bisect_right(db.tx_counts, tx_num))
+            self.assertEqual(support['validatheight'], valid_height)
+
+        self.assertEqual(total_es_amount, es_support_amount)
+        self.assertEqual(active_es_amount, total_lbrycrd_amount)
 
     async def assertMatchClaimsForName(self, name):
         expected = json.loads(await self.blockchain._cli_cmnd('getclaimsforname', name, "", "true"))
@@ -153,7 +165,7 @@ class BaseResolveTestCase(CommandTestCase):
             self.assertEqual(claim_from_es[0][0]['claim_hash'][::-1].hex(), claim_id)
             self.assertMatchESClaim(claim_from_es[0][0], claim)
             self._check_supports(claim_id, c.get('supports', []),
-                                 claim_from_es[0][0]['support_amount'], c['effectiveamount'] > 0)
+                                 claim_from_es[0][0]['support_amount'])
 
     async def assertNameState(self, height: int, name: str, winning_claim_id: str, last_takeover_height: int,
                                non_winning_claims: List[ClaimStateValue]):
@@ -458,16 +470,16 @@ class ResolveCommand(BaseResolveTestCase):
         self.assertEqual(one, claim6['name'])
 
     async def test_resolve_old_claim(self):
-        channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
+        channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0', blocking=True)
         await self.confirm_tx(channel.id)
         address = channel.outputs[0].get_address(self.account.ledger)
         claim = generate_signed_legacy(address, channel.outputs[0])
         tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account)
         await tx.sign([self.account])
-        await self.broadcast(tx)
+        await self.broadcast_and_confirm(tx)
-        await self.confirm_tx(tx.id)
 
         response = await self.resolve('@olds/example')
+        self.assertTrue('is_channel_signature_valid' in response, str(response))
         self.assertTrue(response['is_channel_signature_valid'])
 
         claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature))
@@ -475,8 +487,7 @@ class ResolveCommand(BaseResolveTestCase):
             'bad_example', claim.SerializeToString(), 1, address, [self.account], self.account
         )
         await tx.sign([self.account])
-        await self.broadcast(tx)
+        await self.broadcast_and_confirm(tx)
-        await self.confirm_tx(tx.id)
 
         response = await self.resolve('bad_example')
         self.assertFalse(response['is_channel_signature_valid'])

tests/integration/transactions/test_transaction_commands.py

@@ -19,7 +19,7 @@ class TransactionCommandsTestCase(CommandTestCase):
     async def test_transaction_show(self):
         # local tx
         result = await self.out(self.daemon.jsonrpc_account_send(
-            '5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
+            '5.0', await self.daemon.jsonrpc_address_unused(self.account.id), blocking=True
         ))
         await self.confirm_tx(result['txid'])
         tx = await self.daemon.jsonrpc_transaction_show(result['txid'])

tests/integration/transactions/test_transactions.py

@@ -1,6 +1,7 @@
 import asyncio
 import random
 
+import lbry.wallet.rpc.jsonrpc
 from lbry.wallet.transaction import Transaction, Output, Input
 from lbry.testcase import IntegrationTestCase
 from lbry.wallet.util import satoshis_to_coins, coins_to_satoshis
@@ -198,7 +199,7 @@ class BasicTransactionTests(IntegrationTestCase):
         other_account = self.wallet.generate_account(self.ledger)
         other_address = await other_account.receiving.get_or_create_usable_address()
         self.ledger.coin_selection_strategy = 'sqlite'
-        await self.ledger.subscribe_account(self.account)
+        await self.ledger.subscribe_account(other_account)
 
         accepted = asyncio.ensure_future(self.on_address_update(address))
         _ = await self.send_to_address_and_wait(address, 1.0)
@@ -258,6 +259,8 @@ class BasicTransactionTests(IntegrationTestCase):
         async def broadcast(tx):
             try:
                 return await real_broadcast(tx)
+            except lbry.wallet.rpc.jsonrpc.RPCError:
+                pass
             finally:
                 e.set()