From c5f18a416696178f57ac3e8ec1a526c9a41ae9a2 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 6 May 2022 11:54:25 -0400 Subject: [PATCH] expose `--address_history_cache_size` setting for `scribe` --- README.md | 1 + scribe/blockchain/env.py | 9 ++++++++- scribe/blockchain/service.py | 17 ++++------------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 91180bf..fa06aab 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ For various reasons it may be desirable to block or filtering content from claim #### Options for `scribe` - `--db_max_open_files` This setting translates into the max_open_files option given to rocksdb. A higher number will use more memory. Defaults to 64. + - `--address_history_cache_size` The count of items in the address history cache used for processing blocks and mempool updates. A higher number will use more memory, shouldn't ever need to be higher than 10000. Defaults to 1000. #### Options for `scribe-elastic-sync` - `--reindex` If this flag is set drop and rebuild the elasticsearch index. diff --git a/scribe/blockchain/env.py b/scribe/blockchain/env.py index ee07519..7a7fc09 100644 --- a/scribe/blockchain/env.py +++ b/scribe/blockchain/env.py @@ -5,11 +5,13 @@ class BlockchainEnv(Env): def __init__(self, db_dir=None, max_query_workers=None, chain=None, reorg_limit=None, prometheus_port=None, cache_all_tx_hashes=None, cache_all_claim_txos=None, blocking_channel_ids=None, filtering_channel_ids=None, - db_max_open_files=64, daemon_url=None): + db_max_open_files=64, daemon_url=None, hashX_history_cache_size=None): super().__init__(db_dir, max_query_workers, chain, reorg_limit, prometheus_port, cache_all_tx_hashes, cache_all_claim_txos, blocking_channel_ids, filtering_channel_ids) self.db_max_open_files = db_max_open_files self.daemon_url = daemon_url if daemon_url is not None else self.required('DAEMON_URL') + self.hashX_history_cache_size = hashX_history_cache_size if hashX_history_cache_size is not None \ + else self.integer('ADDRESS_HISTORY_CACHE_SIZE', 1000) @classmethod def contribute_to_arg_parser(cls, parser): @@ -22,6 +24,11 @@ class BlockchainEnv(Env): parser.add_argument('--db_max_open_files', type=int, default=64, help='This setting translates into the max_open_files option given to rocksdb. ' 'A higher number will use more memory. Defaults to 64.') + parser.add_argument('--address_history_cache_size', type=int, + default=cls.integer('ADDRESS_HISTORY_CACHE_SIZE', 1000), + help="LRU cache size for address histories, used when processing new blocks " + "and when processing mempool updates. Can be set in env with " + "'ADDRESS_HISTORY_CACHE_SIZE'") @classmethod def from_arg_parser(cls, args): diff --git a/scribe/blockchain/service.py b/scribe/blockchain/service.py index 865c859..a3a4dc8 100644 --- a/scribe/blockchain/service.py +++ b/scribe/blockchain/service.py @@ -50,25 +50,16 @@ class BlockchainProcessorService(BlockchainService): self.coin = env.coin self.wait_for_blocks_duration = 0.1 self._ready_to_stop = asyncio.Event() - + self.blocks_event = asyncio.Event() + self.prefetcher = Prefetcher(self.daemon, env.coin, self.blocks_event) self._caught_up_event: Optional[asyncio.Event] = None self.height = 0 self.tip = bytes.fromhex(self.coin.GENESIS_HASH)[::-1] self.tx_count = 0 - self.blocks_event = asyncio.Event() - self.prefetcher = Prefetcher(self.daemon, env.coin, self.blocks_event) - # self.logger = logging.getLogger(__name__) - - # Meta self.touched_hashXs: Set[bytes] = set() - - # UTXO cache self.utxo_cache: Dict[Tuple[bytes, int], Tuple[bytes, int]] = {} - # Claimtrie cache - self.db_op_stack: Optional['RevertableOpStack'] = None - ################################# # attributes used for calculating stake activations and takeovers per block ################################# @@ -125,8 +116,8 @@ class BlockchainProcessorService(BlockchainService): self.pending_transaction_num_mapping: Dict[bytes, int] = {} self.pending_transactions: Dict[int, bytes] = {} - self.hashX_history_cache = LRUCache(1000) - self.hashX_full_cache = LRUCache(1000) + self.hashX_history_cache = LRUCache(min(100, max(0, env.hashX_history_cache_size))) + self.hashX_full_cache = LRUCache(min(100, max(0, env.hashX_history_cache_size))) async def run_in_thread_with_lock(self, func, *args): # Run in a thread to prevent blocking. Shielded so that