LBRYCommunity/hub
1
0
Fork 0
Code Issues 35 Pull requests 6 Projects Releases Packages Wiki Activity

check for scribe needing a restart due to low memory

Browse source
This commit is contained in:
Jack Robison 2023-02-07 17:07:30 -05:00
parent 8147bbf3b9
commit f2c494d4d6
2 changed files with 26 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
hub/db/db.py
View file

@ -49,6 +49,7 @@ class SecondaryDB:
self._reorg_limit = reorg_limit self._reorg_limit = reorg_limit
self._cache_all_tx_hashes = cache_all_tx_hashes self._cache_all_tx_hashes = cache_all_tx_hashes
self._secondary_name = secondary_name self._secondary_name = secondary_name
self._need_restart_path = os.path.join(self._db_dir, 'NEED_SCRIBE_RESTART')
if secondary_name: if secondary_name:
assert max_open_files == -1, 'max open files must be -1 for secondary readers' assert max_open_files == -1, 'max open files must be -1 for secondary readers'
self._db_max_open_files = max_open_files self._db_max_open_files = max_open_files
@ -1000,6 +1001,27 @@ class SecondaryDB:
secondary_path = '' if not self._secondary_name else os.path.join( secondary_path = '' if not self._secondary_name else os.path.join(
self._db_dir, self._secondary_name self._db_dir, self._secondary_name
) )
open_db_canary = None
if self._secondary_name:
open_db_canary = os.path.join(self._db_dir, f'{self._secondary_name}-db-canary')
if os.path.exists(open_db_canary):
with open(self._need_restart_path, 'w+') as f:
f.write(f"{time.strftime(f'%Y-%m-%d %H:%M:%S')} {self._secondary_name}\n")
raise RuntimeError('scribe restart is needed')
else:
with open(open_db_canary, 'w'):
pass
else:
herald_db_canary = os.path.join(self._db_dir, 'lbry-reader-db-canary')
es_sync_db_canary = os.path.join(self._db_dir, 'lbry-elastic-writer-db-canary')
if os.path.exists(herald_db_canary):
os.remove(herald_db_canary)
if os.path.exists(es_sync_db_canary):
os.remove(es_sync_db_canary)
if os.path.exists(self._need_restart_path):
os.remove(self._need_restart_path)
db_path = os.path.join(self._db_dir, 'lbry-rocksdb') db_path = os.path.join(self._db_dir, 'lbry-rocksdb')
self.prefix_db = PrefixDB( self.prefix_db = PrefixDB(
db_path, reorg_limit=self._reorg_limit, max_open_files=self._db_max_open_files, db_path, reorg_limit=self._reorg_limit, max_open_files=self._db_max_open_files,
@ -1008,6 +1030,7 @@ class SecondaryDB:
) )
if secondary_path != '': if secondary_path != '':
os.remove(open_db_canary)
self.logger.info(f'opened db for read only: lbry-rocksdb (%s)', db_path) self.logger.info(f'opened db for read only: lbry-rocksdb (%s)', db_path)
else: else:
self.logger.info(f'opened db for writing: lbry-rocksdb (%s)', db_path) self.logger.info(f'opened db for writing: lbry-rocksdb (%s)', db_path)

3
hub/scribe/service.py
View file

@ -1,3 +1,4 @@
import os
import time import time
import asyncio import asyncio
import typing import typing
@ -2069,6 +2070,8 @@ class BlockchainProcessorService(BlockchainService):
"""Loop forever processing blocks as they arrive.""" """Loop forever processing blocks as they arrive."""
self._caught_up_event = caught_up_event self._caught_up_event = caught_up_event
try: try:
if os.path.exists(self.db._need_restart_path):
raise MemoryError()
if self.height != self.daemon.cached_height() and not self.db.catching_up: if self.height != self.daemon.cached_height() and not self.db.catching_up:
await self._need_catch_up() # tell the readers that we're still catching up with lbrycrd/lbcd await self._need_catch_up() # tell the readers that we're still catching up with lbrycrd/lbcd
while not self._stopping: while not self._stopping: