From bd06d1c992e25361b576489a5638007a9b929753 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Tue, 31 Mar 2020 13:57:30 -0400 Subject: [PATCH] Improves database configuration --- setup.py | 7 ++-- src/database/models.py | 13 +------- src/main.py | 12 +++++-- src/server/app.py | 72 ++++++++++++++++++++++-------------------- test/test_database.py | 2 +- test/test_server.py | 26 +++++++-------- test/testcase.py | 1 - 7 files changed, 65 insertions(+), 68 deletions(-) diff --git a/setup.py b/setup.py index 25087e4..c8755cf 100644 --- a/setup.py +++ b/setup.py @@ -17,12 +17,11 @@ setup( 'mysql-connector-python', 'pyyaml', 'Faker>=1.0.7', - 'asyncio>=3.4.3', - 'aiohttp==3.5.4', - 'aiojobs==0.2.2', + 'asyncio', + 'aiohttp', + 'aiojobs', 'ecdsa>=0.13.3', 'cryptography==2.5', - 'aiosqlite==0.10.0', 'PyNaCl>=1.3.0', 'requests', 'cython', diff --git a/src/database/models.py b/src/database/models.py index 43a1bbb..f999f34 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -12,14 +12,6 @@ from src.server.validation import is_valid_base_comment from src.misc import clean -def get_database_connection(dbms, db_name, **params): - if dbms == 'mysql': - return MySQLDatabase(db_name, **params) - else: - # return SqliteDatabase('/home/oleg/PycharmProjects/comment-server/database/default_pw.db') - return SqliteDatabase(db_name) - - class Channel(Model): claim_id = TextField(column_name='ClaimId', primary_key=True) name = TextField(column_name='Name') @@ -157,7 +149,7 @@ def create_comment(comment: str = None, claim_id: str = None, raise ValueError('Invalid Parameters given for comment') channel, _ = Channel.get_or_create(name=channel_name, claim_id=channel_id) - if parent_id: + if parent_id and not claim_id: parent: Comment = Comment.get_by_id(parent_id) claim_id = parent.claim_id @@ -219,7 +211,4 @@ if __name__ == '__main__': (Comment.claim_id ** '420%')) ) - ids = get_comment_ids('4207d2378bf4340e68c9d88faf7ee24ea1a1f95a') - print(json.dumps(comments, indent=4)) - print(json.dumps(ids, indent=4)) diff --git a/src/main.py b/src/main.py index c22a82b..124c7d6 100644 --- a/src/main.py +++ b/src/main.py @@ -84,12 +84,13 @@ def get_config(filepath): def setup_db_from_config(config: dict): - if 'sqlite' in config['database']: + mode = config['mode'] + if config[mode]['database'] == 'sqlite': if not os.path.exists(DATABASE_DIR): os.mkdir(DATABASE_DIR) - config['db_path'] = os.path.join( - DATABASE_DIR, config['database']['sqlite'] + config[mode]['db_file'] = os.path.join( + DATABASE_DIR, config[mode]['name'] ) @@ -98,10 +99,15 @@ def main(argv=None): parser = argparse.ArgumentParser(description='LBRY Comment Server') parser.add_argument('--port', type=int) parser.add_argument('--config', type=str) + parser.add_argument('--mode', type=str) args = parser.parse_args(argv) config = get_config(CONFIG_FILE) if not args.config else args.config setup_logging_from_config(config) + + if args.mode: + config['mode'] = args.mode + setup_db_from_config(config) if args.port: diff --git a/src/server/app.py b/src/server/app.py index 35471fd..389083e 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -1,7 +1,6 @@ # cython: language_level=3 import asyncio import logging -import pathlib import signal import time @@ -9,61 +8,67 @@ import aiojobs import aiojobs.aiohttp from aiohttp import web -from src.database.queries import obtain_connection, DatabaseWriter -from src.database.queries import setup_database +from peewee import * from src.server.handles import api_endpoint, get_api_endpoint +from src.database.models import Comment, Channel +MODELS = [Comment, Channel] logger = logging.getLogger(__name__) -async def setup_db_schema(app): - if not pathlib.Path(app['db_path']).exists(): - logger.info(f'Setting up schema in {app["db_path"]}') - setup_database(app['db_path']) - else: - logger.info(f'Database already exists in {app["db_path"]}, skipping setup') +def setup_database(app): + config = app['config'] + mode = config['mode'] + + # switch between Database objects + if config[mode]['database'] == 'mysql': + app['db'] = MySQLDatabase( + database=config[mode]['name'], + user=config[mode]['user'], + host=config[mode]['host'], + password=config[mode]['password'], + port=config[mode]['port'], + ) + elif config[mode]['database'] == 'sqlite': + app['db'] = SqliteDatabase( + config[mode]['file'], + pragmas=config[mode]['pragmas'] + ) + + # bind the Model list to the database + app['db'].bind(MODELS, bind_refs=False, bind_backrefs=False) async def start_background_tasks(app): - # Reading the DB - app['reader'] = obtain_connection(app['db_path'], True) - - # Scheduler to prevent multiple threads from writing to DB simulataneously - app['comment_scheduler'] = await aiojobs.create_scheduler(limit=1, pending_limit=0) - app['db_writer'] = DatabaseWriter(app['db_path']) - app['writer'] = app['db_writer'].connection + app['db'].connect() + app['db'].create_tables(MODELS) # for requesting to external and internal APIs app['webhooks'] = await aiojobs.create_scheduler(pending_limit=0) async def close_database_connections(app): - app['reader'].close() - app['writer'].close() - app['db_writer'].cleanup() + app['db'].close() async def close_schedulers(app): - logger.info('Closing comment_scheduler') - await app['comment_scheduler'].close() - logger.info('Closing scheduler for webhook requests') await app['webhooks'].close() class CommentDaemon: - def __init__(self, config, db_file=None, **kwargs): + def __init__(self, config, **kwargs): app = web.Application() + app['config'] = config # configure the config - app['config'] = config - self.config = app['config'] + self.config = config + self.host = config['host'] + self.port = config['port'] - # configure the db file - app['db_path'] = db_file or config.get('db_path') + setup_database(app) # configure the order of tasks to run during app lifetime - app.on_startup.append(setup_db_schema) app.on_startup.append(start_background_tasks) app.on_shutdown.append(close_schedulers) app.on_cleanup.append(close_database_connections) @@ -85,20 +90,19 @@ class CommentDaemon: await self.app_runner.setup() self.app_site = web.TCPSite( runner=self.app_runner, - host=host or self.config['host'], - port=port or self.config['port'], + host=host or self.host, + port=port or self.port, ) await self.app_site.start() - logger.info(f'Comment Server is running on {self.config["host"]}:{self.config["port"]}') + logger.info(f'Comment Server is running on {self.host}:{self.port}') async def stop(self): await self.app_runner.shutdown() await self.app_runner.cleanup() -def run_app(config, db_file=None): - comment_app = CommentDaemon(config=config, db_file=db_file, close_timeout=5.0) - +def run_app(config): + comment_app = CommentDaemon(config=config) loop = asyncio.get_event_loop() def __exit(): diff --git a/test/test_database.py b/test/test_database.py index bf25bba..c698ad1 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -6,7 +6,7 @@ from faker.providers import misc from src.database.models import create_comment from src.database.models import delete_comment -from src.database.models import comment_list, get_comment, get_comments_by_id +from src.database.models import comment_list, get_comment from src.database.models import set_hidden_flag from test.testcase import DatabaseTestCase diff --git a/test/test_server.py b/test/test_server.py index 55fcdba..56bccfb 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -17,6 +17,8 @@ from test.testcase import AsyncioTestCase config = get_config(CONFIG_FILE) +config['mode'] = 'testing' +config['testing']['file'] = ':memory:' if 'slack_webhook' in config: @@ -74,10 +76,10 @@ def create_test_comments(values: iter, **default): class ServerTest(AsyncioTestCase): - db_file = 'test.db' - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + config['mode'] = 'testing' + config['testing']['file'] = ':memory:' self.host = 'localhost' self.port = 5931 @@ -88,11 +90,10 @@ class ServerTest(AsyncioTestCase): @classmethod def tearDownClass(cls) -> None: print('exit reached') - os.remove(cls.db_file) async def asyncSetUp(self): await super().asyncSetUp() - self.server = app.CommentDaemon(config, db_file=self.db_file) + self.server = app.CommentDaemon(config) await self.server.start(host=self.host, port=self.port) self.addCleanup(self.server.stop) @@ -138,14 +139,16 @@ class ServerTest(AsyncioTestCase): test_all = create_test_comments(replace.keys(), **{ k: None for k in replace.keys() }) + test_all.reverse() for test in test_all: - with self.subTest(test=test): + nulls = 'null fields: ' + ', '.join(k for k, v in test.items() if not v) + with self.subTest(test=nulls): message = await self.post_comment(**test) self.assertTrue('result' in message or 'error' in message) if 'error' in message: - self.assertFalse(is_valid_base_comment(**test)) + self.assertFalse(is_valid_base_comment(**test, strict=True)) else: - self.assertTrue(is_valid_base_comment(**test)) + self.assertTrue(is_valid_base_comment(**test, strict=True)) async def test04CreateAllReplies(self): claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8' @@ -223,7 +226,8 @@ class ListCommentsTest(AsyncioTestCase): super().__init__(*args, **kwargs) self.host = 'localhost' self.port = 5931 - self.db_file = 'list_test.db' + config['mode'] = 'testing' + config['testing']['file'] = ':memory:' self.claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8' self.comment_ids = None @@ -234,10 +238,6 @@ class ListCommentsTest(AsyncioTestCase): async def post_comment(self, **params): return await jsonrpc_post(self.url, 'create_comment', **params) - def tearDown(self) -> None: - print('exit reached') - os.remove(self.db_file) - async def create_lots_of_comments(self, n=23): self.comment_list = [{key: self.replace[key]() for key in self.replace.keys()} for _ in range(23)] for comment in self.comment_list: @@ -247,7 +247,7 @@ class ListCommentsTest(AsyncioTestCase): async def asyncSetUp(self): await super().asyncSetUp() - self.server = app.CommentDaemon(config, db_file=self.db_file) + self.server = app.CommentDaemon(config) await self.server.start(self.host, self.port) self.addCleanup(self.server.stop) diff --git a/test/testcase.py b/test/testcase.py index 12e1197..415041d 100644 --- a/test/testcase.py +++ b/test/testcase.py @@ -35,7 +35,6 @@ class DatabaseTestCase(unittest.TestCase): test_db.close() - class AsyncioTestCase(unittest.TestCase): # Implementation inspired by discussion: # https://bugs.python.org/issue32972