From 35c3ff1e30d44affdc2bccc8fead66cdacc9873e Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Thu, 20 Aug 2020 13:31:58 -0400 Subject: [PATCH] test cleaup and fixing --- .github/workflows/main.yml | 2 +- lbry/blockchain/transaction.py | 4 ++++ lbry/db/queries/base.py | 4 ++-- lbry/db/queries/txio.py | 1 + lbry/db/sync.py | 27 ++++++++++++++++++++------- lbry/testcase.py | 4 ++-- tests/unit/db/test_event.py | 5 ++--- 7 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index df2fc9c8b..7b21aa0e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,7 @@ jobs: pip install -e .[test] - env: HOME: /tmp - run: coverage run -m unittest -vv tests.unit.test_conf tests.unit.blockchain tests.unit.test_event_controller tests.unit.crypto tests.unit.schema tests.unit.db + run: coverage run -m unittest -vv tests.unit.test_conf tests.unit.test_console tests.unit.test_event_controller tests/unit/blockchain/test* tests/unit/crypto/test* tests/unit/db/test* tests/unit/schema/test* tests/unit/service/test* # run: coverage run -m unittest discover -vv tests.unit - env: COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/lbry/blockchain/transaction.py b/lbry/blockchain/transaction.py index 8025719b4..7ffbcb494 100644 --- a/lbry/blockchain/transaction.py +++ b/lbry/blockchain/transaction.py @@ -54,6 +54,10 @@ class TXRefMutable(TXRef): def height(self): return self.tx.height + @property + def timestamp(self): + return self.tx.timestamp + def reset(self): self._id = None self._hash = None diff --git a/lbry/db/queries/base.py b/lbry/db/queries/base.py index 368e3c0b2..cd4716b14 100644 --- a/lbry/db/queries/base.py +++ b/lbry/db/queries/base.py @@ -2,7 +2,7 @@ from sqlalchemy import text from sqlalchemy.future import select from ..query_context import context -from ..tables import SCHEMA_VERSION, metadata, Version, Claim, Support, Block +from ..tables import SCHEMA_VERSION, metadata, Version, Claim, Support, Block, TX def execute(sql): @@ -30,7 +30,7 @@ def insert_block(block): def insert_transaction(block_hash, tx): - context().get_bulk_loader().add_transaction(block_hash, tx).flush() + context().get_bulk_loader().add_transaction(block_hash, tx).flush(TX) def check_version_and_create_tables(): diff --git a/lbry/db/queries/txio.py b/lbry/db/queries/txio.py index 307da8834..05f680b1c 100644 --- a/lbry/db/queries/txio.py +++ b/lbry/db/queries/txio.py @@ -384,6 +384,7 @@ def rows_to_txos(rows: List[dict], include_tx=True) -> List[Output]: if row['tx_hash'] not in tx_cache: tx_cache[row['tx_hash']] = Transaction( row['raw'], height=row['height'], position=row['tx_position'], + timestamp=row['timestamp'], is_verified=bool(row['is_verified']), ) txo = tx_cache[row['tx_hash']].outputs[row['txo_position']] diff --git a/lbry/db/sync.py b/lbry/db/sync.py index 01beefa93..32c7d4319 100644 --- a/lbry/db/sync.py +++ b/lbry/db/sync.py @@ -1,9 +1,10 @@ from sqlalchemy.future import select from lbry.db.query_context import progress, Event -from lbry.db.tables import TXI, TXO, Claim, Support +from lbry.db.tables import TX, TXI, TXO, Claim, Support from .constants import TXO_TYPES, CLAIM_TYPE_CODES from .queries import ( + BASE_SELECT_TXO_COLUMNS, rows_to_txos, where_unspent_txos, where_abandoned_supports, where_abandoned_claims @@ -22,9 +23,9 @@ SUPPORT_DELETE_EVENT = Event.add("client.sync.supports.delete", "supports") def process_all_things_after_sync(): with progress(SPENDS_UPDATE_EVENT) as p: p.start(2) - set_input_addresses(p.ctx) - p.step(1) update_spent_outputs(p.ctx) + p.step(1) + set_input_addresses(p.ctx) p.step(2) with progress(SUPPORT_DELETE_EVENT) as p: p.start(1) @@ -32,7 +33,11 @@ def process_all_things_after_sync(): p.ctx.execute(sql) with progress(SUPPORT_INSERT_EVENT) as p: loader = p.ctx.get_bulk_loader() - sql = where_unspent_txos(TXO_TYPES['support'], missing_in_supports_table=True) + sql = ( + select(*BASE_SELECT_TXO_COLUMNS) + .where(where_unspent_txos(TXO_TYPES['support'], missing_in_supports_table=True)) + .select_from(TXO.join(TX)) + ) for support in rows_to_txos(p.ctx.fetchall(sql)): loader.add_support(support) loader.flush(Support) @@ -42,13 +47,21 @@ def process_all_things_after_sync(): p.ctx.execute(sql) with progress(CLAIMS_INSERT_EVENT) as p: loader = p.ctx.get_bulk_loader() - sql = where_unspent_txos(CLAIM_TYPE_CODES, missing_in_claims_table=True) + sql = ( + select(*BASE_SELECT_TXO_COLUMNS) + .where(where_unspent_txos(CLAIM_TYPE_CODES, missing_in_claims_table=True)) + .select_from(TXO.join(TX)) + ) for claim in rows_to_txos(p.ctx.fetchall(sql)): - loader.add_claim(claim) + loader.add_claim(claim, '', 0, 0, 0, 0, staked_support_amount=0, staked_support_count=0) loader.flush(Claim) with progress(CLAIMS_UPDATE_EVENT) as p: loader = p.ctx.get_bulk_loader() - sql = where_unspent_txos(CLAIM_TYPE_CODES, missing_or_stale_in_claims_table=True) + sql = ( + select(*BASE_SELECT_TXO_COLUMNS) + .where(where_unspent_txos(CLAIM_TYPE_CODES, missing_or_stale_in_claims_table=True)) + .select_from(TXO.join(TX)) + ) for claim in rows_to_txos(p.ctx.fetchall(sql)): loader.update_claim(claim) loader.flush(Claim) diff --git a/lbry/testcase.py b/lbry/testcase.py index 32c452b72..95a58a367 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -376,14 +376,14 @@ class UnitDBTestCase(AsyncioTestCase): async def get_txos(self): txos = [] sql = ( - "select txo_hash, txo.position, is_spent from txo join tx using (tx_hash) " + "select txo_hash, txo.position, spent_height from txo join tx using (tx_hash) " "order by tx.height, tx.position, txo.position" ) for txo in await self.db.execute_fetchall(sql): txoid = hexlify(txo["txo_hash"][:32][::-1]).decode() txos.append(( f"{txoid}:{txo['position']}", - bool(txo['is_spent']) + bool(txo['spent_height']) )) return txos diff --git a/tests/unit/db/test_event.py b/tests/unit/db/test_event.py index 982bbadc5..5dcc7e286 100644 --- a/tests/unit/db/test_event.py +++ b/tests/unit/db/test_event.py @@ -6,6 +6,5 @@ from lbry.db.query_context import Event class TestDBEvents(TestCase): def test_enum(self): - self.assertEqual(Event.BLOCK_READ.value, 1) - self.assertEqual(Event.BLOCK_READ.label, "blockchain.sync.block.read") - self.assertEqual(Event(1).label, "blockchain.sync.block.read") + self.assertEqual(Event.get_by_id(1).name, "client.sync.claims.insert") + self.assertEqual(Event.get_by_name("client.sync.claims.insert").id, 1)