faster inputs/outputs sync

lbry/blockchain/sync/steps.py

@@ -5,9 +5,10 @@ from contextvars import ContextVar
 from typing import Set
 
 from sqlalchemy import bindparam, case, distinct, text
+from sqlalchemy.schema import CreateTable
 
 from lbry.db import queries
-from lbry.db.tables import Block as BlockTable
+from lbry.db.tables import Block as BlockTable, TXO, TXI
 from lbry.db.query_context import progress, context, Event
 from lbry.db.queries import rows_to_txos
 from lbry.db.sync import (
@@ -148,40 +149,82 @@ def process_block_save(block_file_number: int, loader, p=None):
 @sync_step(Event.INPUT_UPDATE, initial_sync=True, ongoing_sync=True)
 def process_inputs_outputs(initial_sync=False, p=None):
 
-    step = 1
+    step = 0
-    if initial_sync and p.ctx.is_postgres:
+
-        p.start(6)
+    def next_step():
+        nonlocal step
+        step += 1
+        return step
+
+    if initial_sync:
+        p.start(9)
     else:
         p.start(2)
 
-    # 0. Vacuum
+    if initial_sync:
-    if initial_sync and p.ctx.is_postgres:
+        # A. add tx constraints
-        with p.ctx.engine.connect() as c:
+        if p.ctx.is_postgres:
-            c.execute(text("COMMIT;"))
+            p.ctx.execute(text("ALTER TABLE tx ADD PRIMARY KEY (tx_hash);"))
-            c.execute(text("VACUUM FULL ANALYZE txo;"))
+        p.step(next_step())
-            p.step(step)
-            step += 1
-            c.execute(text("VACUUM FULL ANALYZE txi;"))
-            p.step(step)
-            step += 1
 
-    # 1. Update spent TXOs setting is_spent = True
+    # 1. Update TXIs to have the address of TXO they are spending.
-    update_spent_outputs(p.ctx)
+    if initial_sync:
-    p.step(step)
+        # B. txi table reshuffling
-    step += 1
+        p.ctx.execute(text("ALTER TABLE txi RENAME TO old_txi;"))
-    if initial_sync and p.ctx.is_postgres:
+        p.ctx.execute(CreateTable(TXI, include_foreign_key_constraints=[]))
-        p.ctx.execute(text("ALTER TABLE txo ADD PRIMARY KEY (txo_hash);"))
+        if p.ctx.is_postgres:
-        p.step(step)
+            p.ctx.execute(text("ALTER TABLE txi DROP CONSTRAINT txi_pkey;"))
-        step += 1
+        p.step(next_step())
+        # C. insert
+        old_txi = TXI.alias('old_txi')
+        columns = [c for c in old_txi.columns if c.name != 'address'] + [TXO.c.address]
+        select_txis = select(*columns).select_from(old_txi.join(TXO))
+        insert_txis = TXI.insert().from_select(columns, select_txis)
+        p.ctx.execute(text(
+            str(insert_txis.compile(p.ctx.engine)).replace('txi AS old_txi', 'old_txi')
+        ))
+        p.step(next_step())
+        # D. drop old txi
+        p.ctx.execute(text("DROP TABLE old_txi;"))
+        p.step(next_step())
+        # E. restore integrity constraint
+        if p.ctx.is_postgres:
+            p.ctx.execute(text("ALTER TABLE txi ADD PRIMARY KEY (txo_hash);"))
+        p.step(next_step())
+    else:
+        set_input_addresses(p.ctx)
+        p.step(next_step())
 
-    # 2. Update TXIs to have the address of TXO they are spending.
+    # 2. Update spent TXOs setting is_spent = True
-    set_input_addresses(p.ctx)
+    if initial_sync:
-    p.step(step)
+        # F. txo table reshuffling
-    step += 1
+        p.ctx.execute(text("ALTER TABLE txo RENAME TO old_txo;"))
-    if initial_sync and p.ctx.is_postgres:
+        p.ctx.execute(CreateTable(TXO, include_foreign_key_constraints=[]))
-        p.ctx.execute(text("ALTER TABLE txi ADD PRIMARY KEY (txo_hash);"))
+        if p.ctx.is_postgres:
-        p.step(step)
+            p.ctx.execute(text("ALTER TABLE txo DROP CONSTRAINT txo_pkey;"))
-        step += 1
+        p.step(next_step())
+        # G. insert
+        old_txo = TXO.alias('old_txo')
+        columns = (
+            [c for c in old_txo.columns if c.name != 'is_spent'] +
+            [(TXI.c.txo_hash != None).label('is_spent')]
+        )
+        select_txos = select(*columns).select_from(old_txo.join(TXI, isouter=True))
+        insert_txos = TXO.insert().from_select(columns, select_txos)
+        p.ctx.execute(text(
+            str(insert_txos.compile(p.ctx.engine)).replace('txo AS old_txo', 'old_txo')
+        ))
+        p.step(next_step())
+        # H. drop old txo
+        p.ctx.execute(text("DROP TABLE old_txo;"))
+        p.step(next_step())
+        # I. restore integrity constraint
+        if p.ctx.is_postgres:
+            p.ctx.execute(text("ALTER TABLE txo ADD PRIMARY KEY (txo_hash);"))
+        p.step(next_step())
+    else:
+        update_spent_outputs(p.ctx)
+        p.step(next_step())
 
 
 @sync_step(Event.BLOCK_FILTER, initial_sync=True, ongoing_sync=True)

lbry/db/queries.py

@@ -48,17 +48,18 @@ def check_version_and_create_tables():
         metadata.drop_all(ctx.engine)
         metadata.create_all(ctx.engine)
         ctx.execute(Version.insert().values(version=SCHEMA_VERSION))
-        if ctx.is_postgres:
-            disable_indexes_and_integrity_enforcement()
-
-
-def disable_indexes_and_integrity_enforcement():
-    with context('disable indexes and integrity enforcement (triggers, primary keys, etc)') as ctx:
         for table in metadata.sorted_tables:
-            ctx.execute(text(f"ALTER TABLE {table.name} DISABLE TRIGGER ALL;"))
+            disable_trigger_and_constraints(table.name)
-            if table.name == 'tag':
+
-                continue
+
-            ctx.execute(text(f"ALTER TABLE {table.name} DROP CONSTRAINT {table.name}_pkey CASCADE;"))
+def disable_trigger_and_constraints(table_name):
+    ctx = context()
+    if ctx.is_postgres:
+        ctx.execute(text(f"ALTER TABLE {table_name} DISABLE TRIGGER ALL;"))
+    if table_name == 'tag':
+        return
+    if ctx.is_postgres:
+        ctx.execute(text(f"ALTER TABLE {table_name} DROP CONSTRAINT {table_name}_pkey CASCADE;"))
 
 
 def insert_block(block):

tests/integration/blockchain/test_blockchain.py

@@ -535,17 +535,16 @@ class TestMultiBlockFileSyncing(BasicBlockchainTestCase):
         # 3 - db.sync.input
         self.assertEventsAlmostEqual(
             self.extract_events('db.sync.input', events), [
-                [0, 6],
+                [0, 9],
-                [1, 6],
+                [1, 9],
-                [2, 6],
+                [2, 9],
-                [3, 6],
+                [3, 9],
-                [4, 6],
+                [4, 9],
-                [5, 6],
+                [5, 9],
-                [6, 6],
+                [6, 9],
-            ] if self.db_driver == 'postgresql' else [
+                [7, 9],
-                [0, 2],
+                [8, 9],
-                [1, 2],
+                [9, 9],
-                [2, 2],
             ]
         )
         # 4 - blockchain.sync.claim.meta