From a10eb30771914f3364ba25048c4cf4998efa6ec7 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Mon, 20 Dec 2021 14:46:01 -0500 Subject: [PATCH] fixing stalls in tests --- lbry/testcase.py | 11 +++- lbry/wallet/ledger.py | 4 ++ .../blockchain/test_account_commands.py | 2 +- .../blockchain/test_wallet_commands.py | 15 +++--- .../integration/claims/test_claim_commands.py | 11 ++-- .../datanetwork/test_file_commands.py | 9 ++-- tests/integration/other/test_chris45.py | 12 ++--- .../takeovers/test_resolve_command.py | 51 +++++++++++-------- .../transactions/test_transaction_commands.py | 2 +- .../transactions/test_transactions.py | 5 +- 10 files changed, 73 insertions(+), 49 deletions(-) diff --git a/lbry/testcase.py b/lbry/testcase.py index 470c3f6d2..d328bfc6f 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -265,6 +265,13 @@ class IntegrationTestCase(AsyncioTestCase): def broadcast(self, tx): return self.ledger.broadcast(tx) + async def broadcast_and_confirm(self, tx, ledger=None): + ledger = ledger or self.ledger + notifications = asyncio.create_task(ledger.wait(tx)) + await ledger.broadcast(tx) + await notifications + await self.generate_and_wait(1, [tx.id], ledger) + async def on_header(self, height): if self.ledger.headers.height < height: await self.ledger.on_header.where( @@ -277,7 +284,7 @@ class IntegrationTestCase(AsyncioTestCase): txid = None done = False watcher = (ledger or self.ledger).on_transaction.where( - lambda e: e.tx.id == txid or tx_watch.append(e.tx.id) or done + lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id) ) txid = await self.blockchain.send_to_address(address, amount) @@ -290,7 +297,7 @@ class IntegrationTestCase(AsyncioTestCase): async def generate_and_wait(self, blocks_to_generate, txids, ledger=None): if blocks_to_generate > 0: watcher = (ledger or self.ledger).on_transaction.where( - lambda e: (e.tx.id in txids and txids.remove(e.tx.id)) or len(txids) <= 0 # relies on remove returning None + lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1] # multi-statement lambda ) await self.blockchain.generate(blocks_to_generate) await watcher diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 2b50b5ecd..bf411f08d 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -365,6 +365,10 @@ class Ledger(metaclass=LedgerRegistry): await self.db.close() await self.headers.close() + async def tasks_are_done(self): + await self._update_tasks.done.wait() + await self._other_tasks.done.wait() + @property def local_height_including_downloaded_height(self): return max(self.headers.height, self._download_height) diff --git a/tests/integration/blockchain/test_account_commands.py b/tests/integration/blockchain/test_account_commands.py index 52eb1bfb4..9209fe0c6 100644 --- a/tests/integration/blockchain/test_account_commands.py +++ b/tests/integration/blockchain/test_account_commands.py @@ -103,7 +103,7 @@ class AccountManagement(CommandTestCase): second_account = await self.daemon.jsonrpc_account_create('second account') tx = await self.daemon.jsonrpc_account_send( - '0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id) + '0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id), blocking=True ) await self.confirm_tx(tx.id) await self.assertOutputAmount(['0.05', '9.949876'], utxo_list()) diff --git a/tests/integration/blockchain/test_wallet_commands.py b/tests/integration/blockchain/test_wallet_commands.py index bf1c6735d..17e950425 100644 --- a/tests/integration/blockchain/test_wallet_commands.py +++ b/tests/integration/blockchain/test_wallet_commands.py @@ -73,7 +73,8 @@ class WalletCommands(CommandTestCase): async def test_balance_caching(self): account2 = await self.daemon.jsonrpc_account_create("Tip-er") address2 = await self.daemon.jsonrpc_address_unused(account2.id) - await self.send_to_address_and_wait(address2, 10, 1) + await self.send_to_address_and_wait(address2, 10, 2) + await self.ledger.tasks_are_done() # don't mess with the query count while we need it wallet_balance = self.daemon.jsonrpc_wallet_balance ledger = self.ledger @@ -88,14 +89,16 @@ class WalletCommands(CommandTestCase): self.assertIsNone(ledger._balance_cache.get(self.account.id)) query_count += 2 - self.assertEqual(await wallet_balance(), expected) + balance = await wallet_balance() self.assertEqual(self.ledger.db.db.query_count, query_count) + self.assertEqual(balance, expected) self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0') self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0') # calling again uses cache - self.assertEqual(await wallet_balance(), expected) + balance = await wallet_balance() self.assertEqual(self.ledger.db.db.query_count, query_count) + self.assertEqual(balance, expected) self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0') self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0') @@ -151,7 +154,7 @@ class WalletCommands(CommandTestCase): address2 = await self.daemon.jsonrpc_address_unused(account2.id) # send lbc to someone else - tx = await self.daemon.jsonrpc_account_send('1.0', address2) + tx = await self.daemon.jsonrpc_account_send('1.0', address2, blocking=True) await self.confirm_tx(tx.id) self.assertEqual(await account_balance(), { 'total': '8.97741', @@ -184,7 +187,7 @@ class WalletCommands(CommandTestCase): }) # tip claimed - tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0) + tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0, blocking=True) await self.confirm_tx(tx.id) self.assertEqual(await account_balance(), { 'total': '9.277303', @@ -290,7 +293,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase): '3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1' '66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16' 'a97a6d6a4a8effd29d748901bb9789352519cd00b13d' - ), self.daemon2) + ), self.daemon2, blocking=True) await self.confirm_tx(channel['txid'], self.daemon2.ledger) # both daemons will have the channel but only one has the cert so far diff --git a/tests/integration/claims/test_claim_commands.py b/tests/integration/claims/test_claim_commands.py index 2ada793bf..e9d1487cb 100644 --- a/tests/integration/claims/test_claim_commands.py +++ b/tests/integration/claims/test_claim_commands.py @@ -494,8 +494,7 @@ class ClaimSearchCommand(ClaimTestCase): tx = await Transaction.claim_create( 'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account) await tx.sign([self.account]) - await self.broadcast(tx) - await self.confirm_tx(tx.id) + await self.broadcast_and_confirm(tx) octet = await self.stream_create() video = await self.stream_create('chrome', file_path=self.video_file_name) @@ -1226,7 +1225,7 @@ class ChannelCommands(CommandTestCase): data_to_sign = "CAFEBABE" # claim new name await self.channel_create('@someotherchan') - channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1') + channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1', blocking=True) await self.confirm_tx(channel_tx.id) channel = channel_tx.outputs[0] signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign)) @@ -1373,7 +1372,7 @@ class StreamCommands(ClaimTestCase): self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available']) result = await self.out(self.daemon.jsonrpc_account_send( - '5.0', await self.daemon.jsonrpc_address_unused(account2_id) + '5.0', await self.daemon.jsonrpc_address_unused(account2_id), blocking=True )) await self.confirm_tx(result['txid']) @@ -2177,7 +2176,7 @@ class SupportCommands(CommandTestCase): tip = await self.out( self.daemon.jsonrpc_support_create( claim_id, '1.0', True, account_id=account2.id, wallet_id='wallet2', - funding_account_ids=[account2.id]) + funding_account_ids=[account2.id], blocking=True) ) await self.confirm_tx(tip['txid']) @@ -2209,7 +2208,7 @@ class SupportCommands(CommandTestCase): support = await self.out( self.daemon.jsonrpc_support_create( claim_id, '2.0', False, account_id=account2.id, wallet_id='wallet2', - funding_account_ids=[account2.id]) + funding_account_ids=[account2.id], blocking=True) ) await self.confirm_tx(support['txid']) diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index c8ec8de60..e7360d316 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -37,8 +37,7 @@ class FileCommands(CommandTestCase): tx_to_update.outputs[0], claim, 1, address, [self.account], self.account ) await tx.sign([self.account]) - await self.broadcast(tx) - await self.confirm_tx(tx.id) + await self.broadcast_and_confirm(tx) self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session self.client_session._session.add_dht_node(('localhost', 4040)) self.client_session.wait_start = False # fixme: this is super slow on tests @@ -506,8 +505,7 @@ class FileCommands(CommandTestCase): tx.outputs[0].claim.stream.fee.address_bytes = b'' tx.outputs[0].script.generate() await tx.sign([self.account]) - await self.broadcast(tx) - await self.confirm_tx(tx.id) + await self.broadcast_and_confirm(tx) async def __raw_value_update_no_fee_amount(self, tx, claim_address): tx = await self.daemon.jsonrpc_stream_update( @@ -517,8 +515,7 @@ class FileCommands(CommandTestCase): tx.outputs[0].claim.stream.fee.message.ClearField('amount') tx.outputs[0].script.generate() await tx.sign([self.account]) - await self.broadcast(tx) - await self.confirm_tx(tx.id) + await self.broadcast_and_confirm(tx) class DiskSpaceManagement(CommandTestCase): diff --git a/tests/integration/other/test_chris45.py b/tests/integration/other/test_chris45.py index bcdbc290b..0e3f35614 100644 --- a/tests/integration/other/test_chris45.py +++ b/tests/integration/other/test_chris45.py @@ -80,7 +80,7 @@ class EpicAdventuresOfChris45(CommandTestCase): # After some soul searching Chris decides that his story needs more # heart and a better ending. He takes down the story and begins the rewrite. - abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=False)) + abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=True)) self.assertEqual(abandon['inputs'][0]['claim_id'], claim_id) await self.confirm_tx(abandon['txid']) @@ -103,7 +103,7 @@ class EpicAdventuresOfChris45(CommandTestCase): # 1 LBC to which Chris readily obliges ramsey_account_id = (await self.out(self.daemon.jsonrpc_account_create("Ramsey")))['id'] ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id) - result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address)) + result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address, blocking=True)) self.assertIn("txid", result) await self.confirm_tx(result['txid']) @@ -133,7 +133,7 @@ class EpicAdventuresOfChris45(CommandTestCase): # And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit # Now to keep this claim winning on the lbry blockchain he immediately supports the claim tx = await self.out(self.daemon.jsonrpc_support_create( - claim_id2, '0.2', account_id=ramsey_account_id + claim_id2, '0.2', account_id=ramsey_account_id, blocking=True )) await self.confirm_tx(tx['txid']) @@ -147,7 +147,7 @@ class EpicAdventuresOfChris45(CommandTestCase): # Now he also wanted to support the original creator of the Award Winning Novel # So he quickly decides to send a tip to him tx = await self.out( - self.daemon.jsonrpc_support_create(claim_id2, '0.3', tip=True, account_id=ramsey_account_id) + self.daemon.jsonrpc_support_create(claim_id2, '0.3', tip=True, account_id=ramsey_account_id, blocking=True) ) await self.confirm_tx(tx['txid']) @@ -158,7 +158,7 @@ class EpicAdventuresOfChris45(CommandTestCase): await self.generate(5) # Seeing the ravishing success of his novel Chris adds support to his claim too - tx = await self.out(self.daemon.jsonrpc_support_create(claim_id2, '0.4')) + tx = await self.out(self.daemon.jsonrpc_support_create(claim_id2, '0.4', blocking=True)) await self.confirm_tx(tx['txid']) # And check if his support showed up @@ -183,7 +183,7 @@ class EpicAdventuresOfChris45(CommandTestCase): # But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris... # Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics - abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=False)) + abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=True)) self.assertTrue(abandon['inputs'][0]['txid'], tx['txid']) await self.confirm_tx(abandon['txid']) diff --git a/tests/integration/takeovers/test_resolve_command.py b/tests/integration/takeovers/test_resolve_command.py index f3fbc5e87..b1346d159 100644 --- a/tests/integration/takeovers/test_resolve_command.py +++ b/tests/integration/takeovers/test_resolve_command.py @@ -1,6 +1,7 @@ import asyncio import json import hashlib +import sys from bisect import bisect_right from binascii import hexlify, unhexlify from collections import defaultdict @@ -91,7 +92,7 @@ class BaseResolveTestCase(CommandTestCase): self.assertEqual(len(claim_from_es[0]), 1) self.assertMatchESClaim(claim_from_es[0][0], claim) self._check_supports(claim.claim_hash.hex(), expected.get('supports', []), - claim_from_es[0][0]['support_amount'], expected['effectiveamount'] > 0) + claim_from_es[0][0]['support_amount']) async def assertMatchClaim(self, name, claim_id, is_active_in_lbrycrd=True): claim = await self.conductor.spv_node.server.bp.db.fs_getclaimbyid(claim_id) @@ -110,7 +111,7 @@ class BaseResolveTestCase(CommandTestCase): expected['claims'][0]['lasttakeoverheight'] = expected['lasttakeoverheight'] self.assertMatchDBClaim(expected['claims'][0], claim) self._check_supports(claim.claim_hash.hex(), expected['claims'][0].get('supports', []), - claim_from_es[0][0]['support_amount'], is_active_in_lbrycrd) + claim_from_es[0][0]['support_amount']) else: if 'claims' in expected and expected['claims'] is not None: # ensure that if we do have the matching claim that it is not active @@ -121,19 +122,30 @@ class BaseResolveTestCase(CommandTestCase): self.assertEqual(claim_id, (await self.assertMatchWinningClaim(name)).claim_hash.hex()) await self.assertMatchClaimsForName(name) - def _check_supports(self, claim_id, lbrycrd_supports, es_support_amount, is_active_in_lbrycrd=True): - total_amount = 0 - db = self.conductor.spv_node.server.db + def _check_supports(self, claim_id, lbrycrd_supports, es_support_amount): + total_lbrycrd_amount = 0.0 + total_es_amount = 0.0 + active_es_amount = 0.0 + db = self.conductor.spv_node.server.bp.db + es_supports = db.get_supports(bytes.fromhex(claim_id)) - for i, (tx_num, position, amount) in enumerate(db.get_supports(bytes.fromhex(claim_id))): - total_amount += amount - if is_active_in_lbrycrd: - support = lbrycrd_supports[i] - self.assertEqual(support['txid'], db.prefix_db.tx_hash.get(tx_num, deserialize_value=False)[::-1].hex()) - self.assertEqual(support['n'], position) - self.assertEqual(support['height'], bisect_right(db.tx_counts, tx_num)) - self.assertEqual(support['validatheight'], db.get_activation(tx_num, position, is_support=True)) - self.assertEqual(total_amount, es_support_amount, f"lbrycrd support amount: {total_amount} vs es: {es_support_amount}") + # we're only concerned about active supports here, and they should match + self.assertTrue(len(es_supports) >= len(lbrycrd_supports)) + + for i, (tx_num, position, amount) in enumerate(es_supports): + total_es_amount += amount + valid_height = db.get_activation(tx_num, position, is_support=True) + if valid_height > db.db_height: + continue + active_es_amount += amount + txid = db.prefix_db.tx_hash.get(tx_num, deserialize_value=False)[::-1].hex() + support = next(filter(lambda s: s['txid'] == txid and s['n'] == position, lbrycrd_supports)) + total_lbrycrd_amount += support['amount'] + self.assertEqual(support['height'], bisect_right(db.tx_counts, tx_num)) + self.assertEqual(support['validatheight'], valid_height) + + self.assertEqual(total_es_amount, es_support_amount) + self.assertEqual(active_es_amount, total_lbrycrd_amount) async def assertMatchClaimsForName(self, name): expected = json.loads(await self.blockchain._cli_cmnd('getclaimsforname', name, "", "true")) @@ -153,7 +165,7 @@ class BaseResolveTestCase(CommandTestCase): self.assertEqual(claim_from_es[0][0]['claim_hash'][::-1].hex(), claim_id) self.assertMatchESClaim(claim_from_es[0][0], claim) self._check_supports(claim_id, c.get('supports', []), - claim_from_es[0][0]['support_amount'], c['effectiveamount'] > 0) + claim_from_es[0][0]['support_amount']) async def assertNameState(self, height: int, name: str, winning_claim_id: str, last_takeover_height: int, non_winning_claims: List[ClaimStateValue]): @@ -458,16 +470,16 @@ class ResolveCommand(BaseResolveTestCase): self.assertEqual(one, claim6['name']) async def test_resolve_old_claim(self): - channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0') + channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0', blocking=True) await self.confirm_tx(channel.id) address = channel.outputs[0].get_address(self.account.ledger) claim = generate_signed_legacy(address, channel.outputs[0]) tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account) await tx.sign([self.account]) - await self.broadcast(tx) - await self.confirm_tx(tx.id) + await self.broadcast_and_confirm(tx) response = await self.resolve('@olds/example') + self.assertTrue('is_channel_signature_valid' in response, str(response)) self.assertTrue(response['is_channel_signature_valid']) claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature)) @@ -475,8 +487,7 @@ class ResolveCommand(BaseResolveTestCase): 'bad_example', claim.SerializeToString(), 1, address, [self.account], self.account ) await tx.sign([self.account]) - await self.broadcast(tx) - await self.confirm_tx(tx.id) + await self.broadcast_and_confirm(tx) response = await self.resolve('bad_example') self.assertFalse(response['is_channel_signature_valid']) diff --git a/tests/integration/transactions/test_transaction_commands.py b/tests/integration/transactions/test_transaction_commands.py index b2994dbdd..fe76dc72d 100644 --- a/tests/integration/transactions/test_transaction_commands.py +++ b/tests/integration/transactions/test_transaction_commands.py @@ -19,7 +19,7 @@ class TransactionCommandsTestCase(CommandTestCase): async def test_transaction_show(self): # local tx result = await self.out(self.daemon.jsonrpc_account_send( - '5.0', await self.daemon.jsonrpc_address_unused(self.account.id) + '5.0', await self.daemon.jsonrpc_address_unused(self.account.id), blocking=True )) await self.confirm_tx(result['txid']) tx = await self.daemon.jsonrpc_transaction_show(result['txid']) diff --git a/tests/integration/transactions/test_transactions.py b/tests/integration/transactions/test_transactions.py index 2088153ca..9fc00b5eb 100644 --- a/tests/integration/transactions/test_transactions.py +++ b/tests/integration/transactions/test_transactions.py @@ -1,6 +1,7 @@ import asyncio import random +import lbry.wallet.rpc.jsonrpc from lbry.wallet.transaction import Transaction, Output, Input from lbry.testcase import IntegrationTestCase from lbry.wallet.util import satoshis_to_coins, coins_to_satoshis @@ -198,7 +199,7 @@ class BasicTransactionTests(IntegrationTestCase): other_account = self.wallet.generate_account(self.ledger) other_address = await other_account.receiving.get_or_create_usable_address() self.ledger.coin_selection_strategy = 'sqlite' - await self.ledger.subscribe_account(self.account) + await self.ledger.subscribe_account(other_account) accepted = asyncio.ensure_future(self.on_address_update(address)) _ = await self.send_to_address_and_wait(address, 1.0) @@ -258,6 +259,8 @@ class BasicTransactionTests(IntegrationTestCase): async def broadcast(tx): try: return await real_broadcast(tx) + except lbry.wallet.rpc.jsonrpc.RPCError: + pass finally: e.set()