expose --address_history_cache_size setting for scribe

README.md

@@ -88,6 +88,7 @@ For various reasons it may be desirable to block or filtering content from claim
 
 #### Options for `scribe`
   - `--db_max_open_files` This setting translates into the max_open_files option given to rocksdb. A higher number will use more memory. Defaults to 64.
+  - `--address_history_cache_size` The count of items in the address history cache used for processing blocks and mempool updates. A higher number will use more memory, shouldn't ever need to be higher than 10000. Defaults to 1000.
 
 #### Options for `scribe-elastic-sync`
   - `--reindex` If this flag is set drop and rebuild the elasticsearch index.

scribe/blockchain/env.py

@@ -5,11 +5,13 @@ class BlockchainEnv(Env):
     def __init__(self, db_dir=None, max_query_workers=None, chain=None, reorg_limit=None,
                  prometheus_port=None, cache_all_tx_hashes=None, cache_all_claim_txos=None,
                  blocking_channel_ids=None, filtering_channel_ids=None,
-                 db_max_open_files=64, daemon_url=None):
+                 db_max_open_files=64, daemon_url=None, hashX_history_cache_size=None):
         super().__init__(db_dir, max_query_workers, chain, reorg_limit, prometheus_port, cache_all_tx_hashes,
                          cache_all_claim_txos, blocking_channel_ids, filtering_channel_ids)
         self.db_max_open_files = db_max_open_files
         self.daemon_url = daemon_url if daemon_url is not None else self.required('DAEMON_URL')
+        self.hashX_history_cache_size = hashX_history_cache_size if hashX_history_cache_size is not None \
+            else self.integer('ADDRESS_HISTORY_CACHE_SIZE', 1000)
 
     @classmethod
     def contribute_to_arg_parser(cls, parser):
@@ -22,6 +24,11 @@ class BlockchainEnv(Env):
         parser.add_argument('--db_max_open_files', type=int, default=64,
                             help='This setting translates into the max_open_files option given to rocksdb. '
                                  'A higher number will use more memory. Defaults to 64.')
+        parser.add_argument('--address_history_cache_size', type=int,
+                            default=cls.integer('ADDRESS_HISTORY_CACHE_SIZE', 1000),
+                            help="LRU cache size for address histories, used when processing new blocks "
+                                 "and when processing mempool updates. Can be set in env with "
+                                 "'ADDRESS_HISTORY_CACHE_SIZE'")
 
     @classmethod
     def from_arg_parser(cls, args):

scribe/blockchain/service.py

@@ -50,25 +50,16 @@ class BlockchainProcessorService(BlockchainService):
         self.coin = env.coin
         self.wait_for_blocks_duration = 0.1
         self._ready_to_stop = asyncio.Event()
-
+        self.blocks_event = asyncio.Event()
+        self.prefetcher = Prefetcher(self.daemon, env.coin, self.blocks_event)
         self._caught_up_event: Optional[asyncio.Event] = None
         self.height = 0
         self.tip = bytes.fromhex(self.coin.GENESIS_HASH)[::-1]
         self.tx_count = 0
 
-        self.blocks_event = asyncio.Event()
-        self.prefetcher = Prefetcher(self.daemon, env.coin, self.blocks_event)
-        # self.logger = logging.getLogger(__name__)
-
-        # Meta
         self.touched_hashXs: Set[bytes] = set()
-
-        # UTXO cache
         self.utxo_cache: Dict[Tuple[bytes, int], Tuple[bytes, int]] = {}
 
-        # Claimtrie cache
-        self.db_op_stack: Optional['RevertableOpStack'] = None
-
         #################################
         # attributes used for calculating stake activations and takeovers per block
         #################################
@@ -125,8 +116,8 @@ class BlockchainProcessorService(BlockchainService):
         self.pending_transaction_num_mapping: Dict[bytes, int] = {}
         self.pending_transactions: Dict[int, bytes] = {}
 
-        self.hashX_history_cache = LRUCache(1000)
+        self.hashX_history_cache = LRUCache(min(100, max(0, env.hashX_history_cache_size)))
-        self.hashX_full_cache = LRUCache(1000)
+        self.hashX_full_cache = LRUCache(min(100, max(0, env.hashX_history_cache_size)))
 
     async def run_in_thread_with_lock(self, func, *args):
         # Run in a thread to prevent blocking.  Shielded so that