From 54a0bf92900938c018225e3070cf1a198758c7b3 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sun, 21 Jun 2020 23:21:43 -0400 Subject: [PATCH] fixups to make lbrynet work on cli with postgres --- lbry/blockchain/database.py | 10 +++++----- lbry/blockchain/sync.py | 5 ++++- lbry/console.py | 8 ++++---- lbry/db/database.py | 4 ++-- lbry/db/queries.py | 2 +- lbry/db/query_context.py | 3 +++ lbry/db/tables.py | 2 +- 7 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lbry/blockchain/database.py b/lbry/blockchain/database.py index 4b1f387ec..f18c6ef13 100644 --- a/lbry/blockchain/database.py +++ b/lbry/blockchain/database.py @@ -9,8 +9,8 @@ from .bcd_data_stream import BCDataStream FILES = [ + 'claims', 'block_index', - 'claims' ] @@ -55,9 +55,9 @@ class BlockchainDB: f"ATTACH DATABASE '{os.path.join(self.directory, file+'.sqlite')}' AS {file}" ) self.connection.create_aggregate("find_shortest_id", 2, FindShortestID) - #self.connection.execute( - # "CREATE INDEX IF NOT EXISTS claim_originalheight_claimid ON claim (originalheight, claimid);" - #) + self.connection.execute("CREATE INDEX IF NOT EXISTS claim_originalheight ON claim (originalheight);") + self.connection.execute("CREATE INDEX IF NOT EXISTS claim_updateheight ON claim (updateheight);") + self.connection.execute("create index IF NOT EXISTS support_blockheight on support (blockheight);") self.connection.row_factory = sqlite3.Row async def open(self): @@ -195,7 +195,7 @@ class BlockchainDB: ) AS shortestID FROM claim WHERE originalHeight BETWEEN ? AND ? - ORDER BY originalHeight, claimid + ORDER BY originalHeight """, (start_height, end_height) return [{ "name": r["name"], diff --git a/lbry/blockchain/sync.py b/lbry/blockchain/sync.py index f2e810fc6..75e97abe9 100644 --- a/lbry/blockchain/sync.py +++ b/lbry/blockchain/sync.py @@ -447,7 +447,8 @@ class BlockchainSync(Sync): if self.on_block_subscription is not None: self.on_block_subscription.cancel() self.db.stop_event.set() - self.advance_loop_task.cancel() + if self.advance_loop_task is not None: + self.advance_loop_task.cancel() async def run(self, f, *args): return await asyncio.get_running_loop().run_in_executor( @@ -498,6 +499,8 @@ class BlockchainSync(Sync): self.db.stop_event.set() for future in pending: future.cancel() + for future in done: + future.result() return best_height_processed = max(f.result() for f in done) # putting event in queue instead of add to progress_controller because diff --git a/lbry/console.py b/lbry/console.py index ff5dc9c4c..b69dead87 100644 --- a/lbry/console.py +++ b/lbry/console.py @@ -71,11 +71,11 @@ class Advanced(Basic): def start_sync_block_bars(self, d): self.bars.clear() - self.get_or_create_bar("parse", "total parsing", "blocks", d['blocks'], True) + self.get_or_create_bar("read", "total reading", "blocks", d['blocks'], True) self.get_or_create_bar("save", "total saving", "txs", d['txs'], True) def close_sync_block_bars(self): - self.bars.pop("parse").close() + self.bars.pop("read").close() self.bars.pop("save").close() def update_sync_block_bars(self, event, d): @@ -106,7 +106,7 @@ class Advanced(Basic): self.start_sync_block_bars(d) elif e.endswith("block.done"): self.close_sync_block_bars() - elif e.endswith("block.parse"): - self.update_sync_block_bars("parse", d) + elif e.endswith("block.read"): + self.update_sync_block_bars("read", d) elif e.endswith("block.save"): self.update_sync_block_bars("save", d) diff --git a/lbry/db/database.py b/lbry/db/database.py index 70beed89d..93d299b32 100644 --- a/lbry/db/database.py +++ b/lbry/db/database.py @@ -78,10 +78,10 @@ class Result(Generic[ResultType]): class Database: - def __init__(self, ledger: 'Ledger', processes=-1): + def __init__(self, ledger: 'Ledger'): self.url = ledger.conf.db_url_or_default self.ledger = ledger - self.processes = self._normalize_processes(processes) + self.processes = self._normalize_processes(ledger.conf.processes) self.executor: Optional[Executor] = None self.message_queue = mp.Queue() self.stop_event = mp.Event() diff --git a/lbry/db/queries.py b/lbry/db/queries.py index bb13033fa..3e5a60e41 100644 --- a/lbry/db/queries.py +++ b/lbry/db/queries.py @@ -54,7 +54,7 @@ def check_version_and_create_tables(): ctx.execute(text("ALTER TABLE txo DISABLE TRIGGER ALL;")) ctx.execute(text("ALTER TABLE tx DISABLE TRIGGER ALL;")) ctx.execute(text("ALTER TABLE claim DISABLE TRIGGER ALL;")) - ctx.execute(text("ALTER TABLE claimtrie DISABLE TRIGGER ALL;")) + ctx.execute(text("ALTER TABLE support DISABLE TRIGGER ALL;")) ctx.execute(text("ALTER TABLE block DISABLE TRIGGER ALL;")) diff --git a/lbry/db/query_context.py b/lbry/db/query_context.py index 355a09d19..507df04ac 100644 --- a/lbry/db/query_context.py +++ b/lbry/db/query_context.py @@ -521,6 +521,9 @@ class BulkLoader: if txo.script.is_claim_name: claim['creation_height'] = tx.height claim['creation_timestamp'] = tx.timestamp + else: + claim['creation_height'] = None + claim['creation_timestamp'] = None self.claims.append(claim) self.tags.extend(tags) return self diff --git a/lbry/db/tables.py b/lbry/db/tables.py index 10a310ba3..3fd5909ef 100644 --- a/lbry/db/tables.py +++ b/lbry/db/tables.py @@ -195,6 +195,6 @@ Support = Table( Takeover = Table( 'takeover', metadata, Column('normalized', Text), - Column('claim_hash', LargeBinary, ForeignKey(TXO.columns.claim_hash)), + Column('claim_hash', LargeBinary), Column('height', Integer), )