From a581425a64988da11377ae964fc4180fdb5adf0a Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 27 Mar 2020 01:26:13 -0400 Subject: [PATCH] Shifts from JSON configuration to yml based --- setup.py | 3 +++ src/definitions.py | 7 +++++++ src/main.py | 42 +++++++++++++++++++++++++++++++++++------- src/server/app.py | 9 +++------ src/settings.py | 17 ----------------- test/test_server.py | 5 ++++- 6 files changed, 52 insertions(+), 31 deletions(-) create mode 100644 src/definitions.py delete mode 100644 src/settings.py diff --git a/setup.py b/setup.py index 923e82a..25087e4 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ setup( data_files=[('config', ['config/conf.json',])], include_package_data=True, install_requires=[ + 'mysql-connector-python', + 'pyyaml', 'Faker>=1.0.7', 'asyncio>=3.4.3', 'aiohttp==3.5.4', @@ -24,5 +26,6 @@ setup( 'PyNaCl>=1.3.0', 'requests', 'cython', + 'peewee' ] ) diff --git a/src/definitions.py b/src/definitions.py new file mode 100644 index 0000000..9972e13 --- /dev/null +++ b/src/definitions.py @@ -0,0 +1,7 @@ +import os + +SRC_DIR = os.path.dirname(os.path.abspath(__file__)) +ROOT_DIR = os.path.dirname(SRC_DIR) +CONFIG_FILE = os.path.join(ROOT_DIR, 'config', 'conf.json') +LOGGING_DIR = os.path.join(ROOT_DIR, 'logs') +DATABASE_DIR = os.path.join(ROOT_DIR, 'database') diff --git a/src/main.py b/src/main.py index b31bcea..c22a82b 100644 --- a/src/main.py +++ b/src/main.py @@ -1,13 +1,20 @@ import argparse +import json +import yaml import logging import logging.config +import os import sys from src.server.app import run_app -from src.settings import config +from src.definitions import LOGGING_DIR, CONFIG_FILE, DATABASE_DIR -def config_logging_from_settings(conf): +def setup_logging_from_config(conf: dict): + # set the logging directory here from the settings file + if not os.path.exists(LOGGING_DIR): + os.mkdir(LOGGING_DIR) + _config = { "version": 1, "disable_existing_loggers": False, @@ -32,7 +39,7 @@ def config_logging_from_settings(conf): "level": "DEBUG", "formatter": "standard", "class": "logging.handlers.RotatingFileHandler", - "filename": conf['path']['debug_log'], + "filename": os.path.join(LOGGING_DIR, 'debug.log'), "maxBytes": 10485760, "backupCount": 5 }, @@ -40,7 +47,7 @@ def config_logging_from_settings(conf): "level": "ERROR", "formatter": "standard", "class": "logging.handlers.RotatingFileHandler", - "filename": conf['path']['error_log'], + "filename": os.path.join(LOGGING_DIR, 'error.log'), "maxBytes": 10485760, "backupCount": 5 }, @@ -48,7 +55,7 @@ def config_logging_from_settings(conf): "level": "NOTSET", "formatter": "aiohttp", "class": "logging.handlers.RotatingFileHandler", - "filename": conf['path']['server_log'], + "filename": os.path.join(LOGGING_DIR, 'server.log'), "maxBytes": 10485760, "backupCount": 5 } @@ -70,15 +77,36 @@ def config_logging_from_settings(conf): logging.config.dictConfig(_config) +def get_config(filepath): + with open(filepath, 'r') as cfile: + config = yaml.load(cfile, Loader=yaml.FullLoader) + return config + + +def setup_db_from_config(config: dict): + if 'sqlite' in config['database']: + if not os.path.exists(DATABASE_DIR): + os.mkdir(DATABASE_DIR) + + config['db_path'] = os.path.join( + DATABASE_DIR, config['database']['sqlite'] + ) + + def main(argv=None): argv = argv or sys.argv[1:] parser = argparse.ArgumentParser(description='LBRY Comment Server') parser.add_argument('--port', type=int) + parser.add_argument('--config', type=str) args = parser.parse_args(argv) - config_logging_from_settings(config) + + config = get_config(CONFIG_FILE) if not args.config else args.config + setup_logging_from_config(config) + setup_db_from_config(config) + if args.port: config['port'] = args.port - config_logging_from_settings(config) + run_app(config) diff --git a/src/server/app.py b/src/server/app.py index a25787b..446c378 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -75,12 +75,9 @@ class CommentDaemon: self.config = app['config'] # configure the db file - if db_file: - app['db_path'] = db_file - app['backup'] = backup - else: - app['db_path'] = config['path']['database'] - app['backup'] = backup or (app['db_path'] + '.backup') + app['db_path'] = db_file or config.get('db_path') + if app['db_path']: + app['backup'] = backup or '.'.join((app['db_path'], 'backup')) # configure the order of tasks to run during app lifetime app.on_startup.append(setup_db_schema) diff --git a/src/settings.py b/src/settings.py deleted file mode 100644 index 2d720c2..0000000 --- a/src/settings.py +++ /dev/null @@ -1,17 +0,0 @@ -# cython: language_level=3 -import json -import pathlib - -root_dir = pathlib.Path(__file__).parent.parent -config_path = root_dir / 'config' / 'conf.json' - - -def get_config(filepath): - with open(filepath, 'r') as cfile: - conf = json.load(cfile) - for key, path in conf['path'].items(): - conf['path'][key] = str(root_dir / path) - return conf - - -config = get_config(config_path) diff --git a/test/test_server.py b/test/test_server.py index 5cd9ea6..55fcdba 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -9,13 +9,16 @@ from faker.providers import internet from faker.providers import lorem from faker.providers import misc -from src.settings import config +from src.main import get_config, CONFIG_FILE from src.server import app from src.server.validation import is_valid_base_comment from test.testcase import AsyncioTestCase +config = get_config(CONFIG_FILE) + + if 'slack_webhook' in config: config.pop('slack_webhook')