moved fast_sync.py script

lbry/scripts/fast_sync.py

@@ -1,276 +0,0 @@
-import os
-import time
-from glob import glob
-import sqlite3
-import asyncio
-import struct
-from contextvars import ContextVar
-from concurrent.futures import ProcessPoolExecutor
-from lbry.wallet.bcd_data_stream import BCDataStream
-from lbry.crypto.hash import double_sha256
-from lbry.wallet.transaction import Transaction
-from binascii import hexlify
-
-
-db = ContextVar('db')
-
-
-def initializer(_path):
-    _db = sqlite3.connect(_path, isolation_level=None, uri=True, timeout=60.0*5)
-    _db.row_factory = sqlite3.Row
-    _db.executescript("""
-    pragma journal_mode=wal;
-    """)
-    db.set(_db)
-
-
-def read_block(block: bytes):
-    reader = BCDataStream()
-    reader.data = block
-
-
-ZERO_BLOCK = bytes((0,)*32)
-
-
-def parse_header(header):
-    version, = struct.unpack('<I', header[:4])
-    timestamp, bits, nonce = struct.unpack('<III', header[100:112])
-    return {
-        'version': version,
-        'block_hash': double_sha256(header),
-        'prev_block_hash': header[4:36],
-        'merkle_root': header[36:68],
-        'claim_trie_root': header[68:100][::-1],
-        'timestamp': timestamp,
-        'bits': bits,
-        'nonce': nonce,
-    }
-
-
-def parse_txs(stream):
-    tx_count = stream.read_compact_size()
-    return [Transaction(position=i)._deserialize(stream) for i in range(tx_count)]
-
-
-def process_file(file_path):
-    print(f'started: {file_path}')
-    sql = db.get()
-    stream = BCDataStream()
-    stream.data = open(file_path, 'rb')
-    blocks, txs, claims, supports, spends = [], [], [], [], []
-    while stream.read_uint32() == 4054508794:
-        block_size = stream.read_uint32()
-        header = parse_header(stream.data.read(112))
-        is_first_block = header['prev_block_hash'] == ZERO_BLOCK
-        for tx in parse_txs(stream):
-            txs.append((header['block_hash'], tx.position, tx.hash))
-            for txi in tx.inputs:
-                if not txi.is_coinbase:
-                    spends.append((header['block_hash'], tx.hash, txi.txo_ref.hash))
-            for output in tx.outputs:
-                try:
-                    if output.is_support:
-                        supports.append((
-                            header['block_hash'], tx.hash, output.ref.hash, output.claim_hash, output.amount
-                        ))
-                    elif output.script.is_claim_name:
-                        claims.append((
-                            header['block_hash'], tx.hash, tx.position, output.ref.hash, output.claim_hash,
-                            output.claim_name, 1, output.amount, None, None
-                        ))
-                    elif output.script.is_update_claim:
-                        claims.append((
-                            header['block_hash'], tx.hash, tx.position, output.ref.hash, output.claim_hash,
-                            output.claim_name, 2, output.amount, None, None
-                        ))
-                except:
-                    pass
-        blocks.append((header['block_hash'], header['prev_block_hash'], 0 if is_first_block else None))
-
-    print(f'inserting sql in {file_path}')
-
-    sql.execute('begin;')
-    sql.executemany("insert into block values (?, ?, ?)", blocks)
-    sql.execute('commit;')
-
-    sql.execute('begin;')
-    sql.executemany("insert into tx values (?, ?, ?)", txs)
-    sql.execute('commit;')
-
-    sql.execute('begin;')
-    sql.executemany("insert into txi values (?, ?, ?)", spends)
-    sql.execute('commit;')
-
-    sql.execute('begin;')
-    sql.executemany("insert into support values (?, ?, ?, ?, ?)", supports)
-    sql.execute('commit;')
-
-    sql.execute('begin;')
-    sql.executemany("insert into claim_history values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", claims)
-    sql.execute('commit;')
-
-    return len(blocks), len(txs)
-
-
-def create_db():
-    sql = db.get()
-    sql.executescript("""
-    create table block (
-        block_hash bytes not null,
-        previous_hash bytes not null,
-        height int
-    );
-    create table tx (
-        block_hash integer not null,
-        position integer not null,
-        tx_hash bytes not null
-    );
-    create table txi (
-        block_hash bytes not null,
-        tx_hash bytes not null,
-        txo_hash bytes not null
-    );
-    create table claim (
-        txo_hash bytes not null,
-        claim_hash bytes not null,
-        claim_name text not null,
-        amount integer not null,
-        height integer
-    );
-    create table claim_history (
-        block_hash bytes not null,
-        tx_hash bytes not null,
-        tx_position integer not null,
-        txo_hash bytes not null,
-        claim_hash bytes not null,
-        claim_name text not null,
-        action integer not null,
-        amount integer not null,
-        height integer,
-        is_spent bool
-    );
-    create table support (
-        block_hash bytes not null,
-        tx_hash bytes not null,
-        txo_hash bytes not null,
-        claim_hash bytes not null,
-        amount integer not null
-    );
-    """)
-
-
-def clean_chain():
-    sql = db.get()
-
-    print('traversing chain and setting height')
-    sql.execute('begin;')
-
-    print(' + adding unique block (block_hash) index')
-    sql.execute("create unique index block_hash_idx on block (block_hash)")
-
-    print(' + adding block (previous_hash) index')
-    sql.execute("create index block_previous_block_hash_idx on block (previous_hash)")
-
-    print(' * setting block.height')
-    sql.execute("""
-    WITH RECURSIVE blocks(block_hash, previous_hash, height) AS (
-        SELECT block_hash, previous_hash, height from block WHERE height = 0
-        UNION
-        SELECT block.block_hash, block.previous_hash, blocks.height+1
-            FROM block, blocks
-            WHERE block.previous_hash=blocks.block_hash
-    )
-    UPDATE block SET height=(SELECT height FROM blocks WHERE block.block_hash=blocks.block_hash)
-    """)
-
-    print(' + adding block (height) index')
-    sql.execute("create index block_height_idx on block (height)")
-
-    for table in ('tx', 'txi', 'claim_history', 'support'):
-        print(f' + adding {table} (block_hash) index')
-        sql.execute(f"create index {table}_block_hash_idx on {table} (block_hash)")
-
-    sql.execute('commit;')
-    sql.execute('begin;')
-
-    print(' - deleting forks:')
-    forks = sql.execute("""
-    WITH RECURSIVE blocks(block_hash, previous_hash) AS (
-        SELECT block_hash, previous_hash FROM block WHERE
-            block_hash NOT IN (SELECT previous_hash FROM block) AND
-            height != (SELECT MAX(height) FROM block)
-        UNION
-        SELECT block.block_hash, block.previous_hash FROM block, blocks WHERE
-            block.block_hash=blocks.previous_hash AND
-            (SELECT COUNT(*) FROM block forks WHERE forks.height=block.height) > 1
-    )
-    SELECT block_hash FROM blocks;
-    """)
-    for i, fork in enumerate(forks):
-        sql.execute('DELETE FROM block WHERE block_hash = ?', (fork[0],))
-        print(f'  - block (fork:{i+1}): {hexlify(fork[0][::-1]).decode()}')
-        deleted_stats = {}
-        for table in ('tx', 'txi', 'claim_history', 'support'):
-            deleted = sql.execute(f"DELETE FROM {table} WHERE block_hash = ?", (fork[0],)).rowcount
-            if deleted > 0:
-                deleted_stats[table] = deleted
-        print(f'   - {deleted_stats}')
-
-    sql.execute('commit;')
-    sql.execute('begin;')
-
-    print(' + adding unique tx (tx_hash, block_hash) index')
-    sql.execute("create unique index tx_hash_idx on tx (tx_hash, block_hash)")
-    print(' + adding unique txi (txo_hash) index')
-    sql.execute("create unique index txi_txo_hash_idx on txi (txo_hash)")
-
-    print('processing claim history & populating claim table')
-
-    print(' * setting claim_history.height and claim_history.is_spent')
-    sql.execute("""
-    UPDATE claim_history SET
-    height = (SELECT height FROM tx JOIN block USING (block_hash) WHERE tx.tx_hash=claim_history.tx_hash),
-    is_spent = COALESCE((SELECT 1 FROM txi WHERE txo_hash=claim_history.txo_hash), 0)
-    """)
-
-    print(' + adding claim_history (claim_hash) index')
-    sql.execute("create index claim_history_hash_idx on claim_history (claim_hash)")
-
-    print(' * populating claim table')
-    sql.execute("""
-    INSERT INTO claim
-    SELECT txo_hash, claim_history.claim_hash, claim_name, amount, height FROM (
-        SELECT claim_hash, is_spent FROM claim_history
-        GROUP BY claim_hash HAVING MAX(height) AND MAX(tx_position)
-    ) AS latest_claim_state JOIN claim_history USING (claim_hash)
-    WHERE latest_claim_state.is_spent=0;
-    """)
-
-    sql.execute('commit;')
-
-
-async def main():
-    db_file = '/tmp/fast_sync.db'
-    if os.path.exists(db_file):
-        os.remove(db_file)
-    initializer(db_file)
-    create_db()
-    executor = ProcessPoolExecutor(
-        4, initializer=initializer, initargs=(db_file,)
-    )
-    file_paths = glob(os.path.join(os.path.expanduser('~/.lbrycrd/blocks/'), 'blk*.dat'))
-    file_paths.sort()
-    total_blocks, total_txs = 0, 0
-    start = time.perf_counter()
-    for file_path, (blocks, txs) in zip(file_paths, executor.map(process_file, file_paths)):
-        print(f"{file_path} {blocks}")
-        total_blocks += blocks
-        total_txs += txs
-    print(f'blocks: {total_blocks} (txs: {total_txs}) in {time.perf_counter()-start}s')
-    print('cleaning chain: set block heights and delete forks')
-    #clean_chain()
-    print(f'done in {time.perf_counter()-start}s')
-    executor.shutdown(True)
-
-if __name__ == '__main__':
-    asyncio.run(main())

scripts/fast_sync.py

@@ -1,19 +1,276 @@
-import os.path
+import os
+import time
+from glob import glob
+import sqlite3
 import asyncio
-from lbry.blockchain import Lbrycrd
+import struct
-from lbry.blockchain.sync import BlockSync
+from contextvars import ContextVar
+from concurrent.futures import ProcessPoolExecutor
+from lbry.wallet.bcd_data_stream import BCDataStream
+from lbry.crypto.hash import double_sha256
+from lbry.wallet.transaction import Transaction
+from binascii import hexlify
+
+
+db = ContextVar('db')
+
+
+def initializer(_path):
+    _db = sqlite3.connect(_path, isolation_level=None, uri=True, timeout=60.0*5)
+    _db.row_factory = sqlite3.Row
+    _db.executescript("""
+    pragma journal_mode=wal;
+    """)
+    db.set(_db)
+
+
+def read_block(block: bytes):
+    reader = BCDataStream()
+    reader.data = block
+
+
+ZERO_BLOCK = bytes((0,)*32)
+
+
+def parse_header(header):
+    version, = struct.unpack('<I', header[:4])
+    timestamp, bits, nonce = struct.unpack('<III', header[100:112])
+    return {
+        'version': version,
+        'block_hash': double_sha256(header),
+        'prev_block_hash': header[4:36],
+        'merkle_root': header[36:68],
+        'claim_trie_root': header[68:100][::-1],
+        'timestamp': timestamp,
+        'bits': bits,
+        'nonce': nonce,
+    }
+
+
+def parse_txs(stream):
+    tx_count = stream.read_compact_size()
+    return [Transaction(position=i)._deserialize(stream) for i in range(tx_count)]
+
+
+def process_file(file_path):
+    print(f'started: {file_path}')
+    sql = db.get()
+    stream = BCDataStream()
+    stream.data = open(file_path, 'rb')
+    blocks, txs, claims, supports, spends = [], [], [], [], []
+    while stream.read_uint32() == 4054508794:
+        block_size = stream.read_uint32()
+        header = parse_header(stream.data.read(112))
+        is_first_block = header['prev_block_hash'] == ZERO_BLOCK
+        for tx in parse_txs(stream):
+            txs.append((header['block_hash'], tx.position, tx.hash))
+            for txi in tx.inputs:
+                if not txi.is_coinbase:
+                    spends.append((header['block_hash'], tx.hash, txi.txo_ref.hash))
+            for output in tx.outputs:
+                try:
+                    if output.is_support:
+                        supports.append((
+                            header['block_hash'], tx.hash, output.ref.hash, output.claim_hash, output.amount
+                        ))
+                    elif output.script.is_claim_name:
+                        claims.append((
+                            header['block_hash'], tx.hash, tx.position, output.ref.hash, output.claim_hash,
+                            output.claim_name, 1, output.amount, None, None
+                        ))
+                    elif output.script.is_update_claim:
+                        claims.append((
+                            header['block_hash'], tx.hash, tx.position, output.ref.hash, output.claim_hash,
+                            output.claim_name, 2, output.amount, None, None
+                        ))
+                except:
+                    pass
+        blocks.append((header['block_hash'], header['prev_block_hash'], 0 if is_first_block else None))
+
+    print(f'inserting sql in {file_path}')
+
+    sql.execute('begin;')
+    sql.executemany("insert into block values (?, ?, ?)", blocks)
+    sql.execute('commit;')
+
+    sql.execute('begin;')
+    sql.executemany("insert into tx values (?, ?, ?)", txs)
+    sql.execute('commit;')
+
+    sql.execute('begin;')
+    sql.executemany("insert into txi values (?, ?, ?)", spends)
+    sql.execute('commit;')
+
+    sql.execute('begin;')
+    sql.executemany("insert into support values (?, ?, ?, ?, ?)", supports)
+    sql.execute('commit;')
+
+    sql.execute('begin;')
+    sql.executemany("insert into claim_history values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", claims)
+    sql.execute('commit;')
+
+    return len(blocks), len(txs)
+
+
+def create_db():
+    sql = db.get()
+    sql.executescript("""
+    create table block (
+        block_hash bytes not null,
+        previous_hash bytes not null,
+        height int
+    );
+    create table tx (
+        block_hash integer not null,
+        position integer not null,
+        tx_hash bytes not null
+    );
+    create table txi (
+        block_hash bytes not null,
+        tx_hash bytes not null,
+        txo_hash bytes not null
+    );
+    create table claim (
+        txo_hash bytes not null,
+        claim_hash bytes not null,
+        claim_name text not null,
+        amount integer not null,
+        height integer
+    );
+    create table claim_history (
+        block_hash bytes not null,
+        tx_hash bytes not null,
+        tx_position integer not null,
+        txo_hash bytes not null,
+        claim_hash bytes not null,
+        claim_name text not null,
+        action integer not null,
+        amount integer not null,
+        height integer,
+        is_spent bool
+    );
+    create table support (
+        block_hash bytes not null,
+        tx_hash bytes not null,
+        txo_hash bytes not null,
+        claim_hash bytes not null,
+        amount integer not null
+    );
+    """)
+
+
+def clean_chain():
+    sql = db.get()
+
+    print('traversing chain and setting height')
+    sql.execute('begin;')
+
+    print(' + adding unique block (block_hash) index')
+    sql.execute("create unique index block_hash_idx on block (block_hash)")
+
+    print(' + adding block (previous_hash) index')
+    sql.execute("create index block_previous_block_hash_idx on block (previous_hash)")
+
+    print(' * setting block.height')
+    sql.execute("""
+    WITH RECURSIVE blocks(block_hash, previous_hash, height) AS (
+        SELECT block_hash, previous_hash, height from block WHERE height = 0
+        UNION
+        SELECT block.block_hash, block.previous_hash, blocks.height+1
+            FROM block, blocks
+            WHERE block.previous_hash=blocks.block_hash
+    )
+    UPDATE block SET height=(SELECT height FROM blocks WHERE block.block_hash=blocks.block_hash)
+    """)
+
+    print(' + adding block (height) index')
+    sql.execute("create index block_height_idx on block (height)")
+
+    for table in ('tx', 'txi', 'claim_history', 'support'):
+        print(f' + adding {table} (block_hash) index')
+        sql.execute(f"create index {table}_block_hash_idx on {table} (block_hash)")
+
+    sql.execute('commit;')
+    sql.execute('begin;')
+
+    print(' - deleting forks:')
+    forks = sql.execute("""
+    WITH RECURSIVE blocks(block_hash, previous_hash) AS (
+        SELECT block_hash, previous_hash FROM block WHERE
+            block_hash NOT IN (SELECT previous_hash FROM block) AND
+            height != (SELECT MAX(height) FROM block)
+        UNION
+        SELECT block.block_hash, block.previous_hash FROM block, blocks WHERE
+            block.block_hash=blocks.previous_hash AND
+            (SELECT COUNT(*) FROM block forks WHERE forks.height=block.height) > 1
+    )
+    SELECT block_hash FROM blocks;
+    """)
+    for i, fork in enumerate(forks):
+        sql.execute('DELETE FROM block WHERE block_hash = ?', (fork[0],))
+        print(f'  - block (fork:{i+1}): {hexlify(fork[0][::-1]).decode()}')
+        deleted_stats = {}
+        for table in ('tx', 'txi', 'claim_history', 'support'):
+            deleted = sql.execute(f"DELETE FROM {table} WHERE block_hash = ?", (fork[0],)).rowcount
+            if deleted > 0:
+                deleted_stats[table] = deleted
+        print(f'   - {deleted_stats}')
+
+    sql.execute('commit;')
+    sql.execute('begin;')
+
+    print(' + adding unique tx (tx_hash, block_hash) index')
+    sql.execute("create unique index tx_hash_idx on tx (tx_hash, block_hash)")
+    print(' + adding unique txi (txo_hash) index')
+    sql.execute("create unique index txi_txo_hash_idx on txi (txo_hash)")
+
+    print('processing claim history & populating claim table')
+
+    print(' * setting claim_history.height and claim_history.is_spent')
+    sql.execute("""
+    UPDATE claim_history SET
+    height = (SELECT height FROM tx JOIN block USING (block_hash) WHERE tx.tx_hash=claim_history.tx_hash),
+    is_spent = COALESCE((SELECT 1 FROM txi WHERE txo_hash=claim_history.txo_hash), 0)
+    """)
+
+    print(' + adding claim_history (claim_hash) index')
+    sql.execute("create index claim_history_hash_idx on claim_history (claim_hash)")
+
+    print(' * populating claim table')
+    sql.execute("""
+    INSERT INTO claim
+    SELECT txo_hash, claim_history.claim_hash, claim_name, amount, height FROM (
+        SELECT claim_hash, is_spent FROM claim_history
+        GROUP BY claim_hash HAVING MAX(height) AND MAX(tx_position)
+    ) AS latest_claim_state JOIN claim_history USING (claim_hash)
+    WHERE latest_claim_state.is_spent=0;
+    """)
+
+    sql.execute('commit;')
 
 
 async def main():
-    chain = Lbrycrd(os.path.expanduser('~/.lbrycrd'), False)
+    db_file = '/tmp/fast_sync.db'
-    sync = BlockSync(chain, use_process_pool=True)
+    if os.path.exists(db_file):
-    if os.path.exists(sync.db.sync_db.db_file_path):
+        os.remove(db_file)
-        os.remove(sync.db.sync_db.db_file_path)
+    initializer(db_file)
-    await sync.db.open()
+    create_db()
-    await sync.load_blocks()
+    executor = ProcessPoolExecutor(
-    #await chain.stop(False)
+        4, initializer=initializer, initargs=(db_file,)
+    )
+    file_paths = glob(os.path.join(os.path.expanduser('~/.lbrycrd/blocks/'), 'blk*.dat'))
+    file_paths.sort()
+    total_blocks, total_txs = 0, 0
+    start = time.perf_counter()
+    for file_path, (blocks, txs) in zip(file_paths, executor.map(process_file, file_paths)):
+        print(f"{file_path} {blocks}")
+        total_blocks += blocks
+        total_txs += txs
+    print(f'blocks: {total_blocks} (txs: {total_txs}) in {time.perf_counter()-start}s')
+    print('cleaning chain: set block heights and delete forks')
+    #clean_chain()
+    print(f'done in {time.perf_counter()-start}s')
+    executor.shutdown(True)
 
-try:
+if __name__ == '__main__':
     asyncio.run(main())
-except KeyboardInterrupt:
-    print('exiting')