From a7bab4a9a0168c6815a522f3d3541527d815a05e Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Tue, 30 Jul 2019 00:14:42 -0400 Subject: [PATCH] Removes unnecessary bloat from config & improves management system --- config/conf.json | 5 +---- src/main.py | 1 + src/server/app.py | 54 +++++++++++++++++++++++------------------------ tests/testcase.py | 18 ++++++++++------ 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/config/conf.json b/config/conf.json index 71e9ebd..7784ab2 100644 --- a/config/conf.json +++ b/config/conf.json @@ -1,10 +1,7 @@ { "PATH": { "SCHEMA": "src/schema/comments_ddl.sql", - "MAIN": "database/comments.db", - "BACKUP": "database/comments.backup.db", - "DEFAULT": "database/default.db", - "TEST": "tests/test.db", + "DATABASE": "database/comments.db", "ERROR_LOG": "logs/error.log", "DEBUG_LOG": "logs/debug.log", "SERVER_LOG": "logs/server.log" diff --git a/src/main.py b/src/main.py index 0a3a9cd..5817a03 100644 --- a/src/main.py +++ b/src/main.py @@ -75,6 +75,7 @@ def main(argv=None): parser = argparse.ArgumentParser(description='LBRY Comment Server') parser.add_argument('--port', type=int) args = parser.parse_args(argv) + config_logging_from_settings(config) if args.port: config['PORT'] = args.port config_logging_from_settings(config) diff --git a/src/server/app.py b/src/server/app.py index 7a54e86..49c0abc 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -18,6 +18,7 @@ logger = logging.getLogger(__name__) async def setup_db_schema(app): + if not pathlib.Path(app['db_path']).exists(): logger.info('Setting up schema in %s', app['db_path']) setup_database(app['db_path'], app['config']['PATH']['SCHEMA']) @@ -25,11 +26,6 @@ async def setup_db_schema(app): logger.info(f'Database already exists in {app["db_path"]}, skipping setup') -async def close_comment_scheduler(app): - logger.info('Closing comment_scheduler') - await app['comment_scheduler'].close() - - async def database_backup_routine(app): try: while True: @@ -43,29 +39,41 @@ async def database_backup_routine(app): async def start_background_tasks(app): app['reader'] = obtain_connection(app['db_path'], True) - app['waitful_backup'] = app.loop.create_task(database_backup_routine(app)) + app['waitful_backup'] = asyncio.create_task(database_backup_routine(app)) app['comment_scheduler'] = await aiojobs.create_scheduler(limit=1, pending_limit=0) app['db_writer'] = DatabaseWriter(app['db_path']) app['writer'] = app['db_writer'].connection -async def stop_background_tasks(app): +async def close_database_connections(app): logger.info('Ending background backup loop') app['waitful_backup'].cancel() await app['waitful_backup'] app['reader'].close() app['writer'].close() + app['db_writer'].cleanup() + + +async def close_comment_scheduler(app): + logger.info('Closing comment_scheduler') + await app['comment_scheduler'].close() class CommentDaemon: - def __init__(self, config, db_path=None, **kwargs): + def __init__(self, config, db_file=None, backup=None, **kwargs): self.config = config app = web.Application() - self.insert_to_config(app, config, db_file=db_path) + app['config'] = config + if db_file: + app['db_path'] = db_file + app['backup'] = backup + else: + app['db_path'] = config['PATH']['DATABASE'] + app['backup'] = backup or (app['db_path'] + '.backup') app.on_startup.append(setup_db_schema) app.on_startup.append(start_background_tasks) - app.on_shutdown.append(stop_background_tasks) app.on_shutdown.append(close_comment_scheduler) + app.on_cleanup.append(close_database_connections) aiojobs.aiohttp.setup(app, **kwargs) app.add_routes([ web.post('/api', api_endpoint), @@ -73,38 +81,30 @@ class CommentDaemon: web.get('/api', get_api_endpoint) ]) self.app = app - self.app_runner = web.AppRunner(app) + self.app_runner = None self.app_site = None - async def start(self): + async def start(self, host=None, port=None): self.app['START_TIME'] = time.time() + self.app_runner = web.AppRunner(self.app) await self.app_runner.setup() self.app_site = web.TCPSite( runner=self.app_runner, - host=self.config['HOST'], - port=self.config['PORT'], + host=host or self.config['HOST'], + port=port or self.config['PORT'], ) await self.app_site.start() logger.info(f'Comment Server is running on {self.config["HOST"]}:{self.config["PORT"]}') async def stop(self): - await self.app.shutdown() - await self.app.cleanup() + await self.app_runner.shutdown() await self.app_runner.cleanup() - @staticmethod - def insert_to_config(app, conf=None, db_file=None): - db_file = db_file if db_file else 'DEFAULT' - app['config'] = conf - app['db_path'] = conf['PATH'][db_file] - app['backup'] = re.sub(r'\.db$', '.backup.db', app['db_path']) - assert app['db_path'] != app['backup'] +def run_app(config, db_file=None): + comment_app = CommentDaemon(config=config, db_file=db_file, close_timeout=5.0) -def run_app(config): - comment_app = CommentDaemon(config=config, db_path='DEFAULT', close_timeout=5.0) - - loop = asyncio.get_event_loop() + loop = asyncio.get_event_loop() def __exit(): raise web.GracefulExit() diff --git a/tests/testcase.py b/tests/testcase.py index 6bf1ca6..4ab4324 100644 --- a/tests/testcase.py +++ b/tests/testcase.py @@ -1,3 +1,4 @@ +import os import pathlib import unittest from asyncio.runners import _cancel_all_tasks # type: ignore @@ -119,15 +120,20 @@ class AsyncioTestCase(unittest.TestCase): class DatabaseTestCase(unittest.TestCase): + db_file = 'test.db' + + def __init__(self, methodName='DatabaseTest'): + super().__init__(methodName) + if pathlib.Path(self.db_file).exists(): + os.remove(self.db_file) + def setUp(self) -> None: super().setUp() - if pathlib.Path(config['PATH']['TEST']).exists(): - teardown_database(config['PATH']['TEST']) - setup_database(config['PATH']['TEST'], config['PATH']['SCHEMA']) - self.conn = obtain_connection(config['PATH']['TEST']) + setup_database(self.db_file, config['PATH']['SCHEMA']) + self.conn = obtain_connection(self.db_file) + self.addCleanup(self.conn.close) + self.addCleanup(os.remove, self.db_file) def tearDown(self) -> None: self.conn.close() - teardown_database(config['PATH']['TEST']) -