From b5f46af3b4ed3f7d112b2f355f7107b8e2173685 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 15 Apr 2022 12:58:31 -0400 Subject: [PATCH] store block hashes in memory --- scribe/blockchain/service.py | 8 +++++--- scribe/db/db.py | 14 ++++++++++++++ scribe/elasticsearch/service.py | 2 +- scribe/service.py | 5 ++++- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/scribe/blockchain/service.py b/scribe/blockchain/service.py index 8a4c410..10b2e7c 100644 --- a/scribe/blockchain/service.py +++ b/scribe/blockchain/service.py @@ -181,7 +181,7 @@ class BlockchainProcessorService(BlockchainService): self.log.warning("failed to get a mempool tx, reorg underway?") return if current_mempool: - if bytes.fromhex(await self.daemon.getbestblockhash())[::-1] != self.coin.header_hash(self.db.headers[-1]): + if bytes.fromhex(await self.daemon.getbestblockhash())[::-1] != self.db.block_hashes[-1]: return await self.run_in_thread( update_mempool, self.db.prefix_db.unsafe_commit, self.db.prefix_db.mempool_tx, _to_put, current_mempool @@ -1417,6 +1417,7 @@ class BlockchainProcessorService(BlockchainService): self.height = height self.db.headers.append(block.header) + self.db.block_hashes.append(self.env.coin.header_hash(block.header)) self.tip = self.coin.header_hash(block.header) self.db.fs_height = self.height @@ -1493,8 +1494,9 @@ class BlockchainProcessorService(BlockchainService): # Check and update self.tip self.db.tx_counts.pop() - reverted_block_hash = self.coin.header_hash(self.db.headers.pop()) - self.tip = self.coin.header_hash(self.db.headers[-1]) + self.db.headers.pop() + reverted_block_hash = self.db.block_hashes.pop() + self.tip = self.db.block_hashes[-1] if self.env.cache_all_tx_hashes: while len(self.db.total_transactions) > self.db.tx_counts[-1]: self.db.tx_num_mapping.pop(self.db.total_transactions.pop()) diff --git a/scribe/db/db.py b/scribe/db/db.py index 6d56c4b..149f8db 100644 --- a/scribe/db/db.py +++ b/scribe/db/db.py @@ -78,6 +78,7 @@ class HubDB: self.tx_counts = None self.headers = None + self.block_hashes = None self.encoded_headers = LRUCacheWithMetrics(1 << 21, metric_name='encoded_headers', namespace='wallet_server') self.last_flush = time.time() @@ -775,6 +776,18 @@ class HubDB: assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}" self.headers = headers + async def _read_block_hashes(self): + def get_block_hashes(): + return [ + block_hash for block_hash in self.prefix_db.block_hash.iterate( + start=(0, ), stop=(self.db_height + 1, ), include_key=False, fill_cache=False, deserialize_value=False + ) + ] + + block_hashes = await asyncio.get_event_loop().run_in_executor(self._executor, get_block_hashes) + assert len(block_hashes) == len(self.headers) + self.block_hashes = block_hashes + async def _read_tx_hashes(self): def _read_tx_hashes(): return list(self.prefix_db.tx_hash.iterate(start=(0,), stop=(self.db_tx_count + 1,), include_key=False, fill_cache=False, deserialize_value=False)) @@ -839,6 +852,7 @@ class HubDB: async def initialize_caches(self): await self._read_tx_counts() await self._read_headers() + await self._read_block_hashes() if self._cache_all_claim_txos: await self._read_claim_txos() if self._cache_all_tx_hashes: diff --git a/scribe/elasticsearch/service.py b/scribe/elasticsearch/service.py index 4f0739e..f22c195 100644 --- a/scribe/elasticsearch/service.py +++ b/scribe/elasticsearch/service.py @@ -231,7 +231,7 @@ class ElasticSyncService(BlockchainReaderService): self._advanced = True def unwind(self): - reverted_block_hash = self.db.coin.header_hash(self.db.headers[-1]) + reverted_block_hash = self.db.block_hashes[-1] super().unwind() packed = self.db.prefix_db.undo.get(len(self.db.tx_counts), reverted_block_hash) touched_or_deleted = None diff --git a/scribe/service.py b/scribe/service.py index a1d7fff..b43ea4f 100644 --- a/scribe/service.py +++ b/scribe/service.py @@ -157,7 +157,9 @@ class BlockchainReaderService(BlockchainService): self.db.total_transactions.append(tx_hash) self.db.tx_num_mapping[tx_hash] = tx_count assert len(self.db.total_transactions) == tx_count, f"{len(self.db.total_transactions)} vs {tx_count}" - self.db.headers.append(self.db.prefix_db.header.get(height, deserialize_value=False)) + header = self.db.prefix_db.header.get(height, deserialize_value=False) + self.db.headers.append(header) + self.db.block_hashes.append(self.env.coin.header_hash(header)) def unwind(self): """ @@ -166,6 +168,7 @@ class BlockchainReaderService(BlockchainService): prev_count = self.db.tx_counts.pop() tx_count = self.db.tx_counts[-1] self.db.headers.pop() + self.db.block_hashes.pop() if self.db._cache_all_tx_hashes: for _ in range(prev_count - tx_count): self.db.tx_num_mapping.pop(self.db.total_transactions.pop())