From 7916d8b7ff12a749167117eadc0793d32d2276ce Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 6 Jan 2020 23:03:59 -0500 Subject: [PATCH 01/46] All comment returning methods now include `claim_id` --- src/database/queries.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/database/queries.py b/src/database/queries.py index 16e6f0b..03815aa 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -12,14 +12,8 @@ from src.database.schema import CREATE_TABLES_QUERY logger = logging.getLogger(__name__) SELECT_COMMENTS_ON_CLAIMS = """ - SELECT comment, comment_id, channel_name, channel_id, channel_url, - timestamp, signature, signing_ts, parent_id, is_hidden - FROM COMMENTS_ON_CLAIMS -""" - -SELECT_COMMENTS_ON_CLAIMS_CLAIMID = """ - SELECT comment, comment_id, claim_id, channel_name, channel_id, channel_url, - timestamp, signature, signing_ts, parent_id, is_hidden + SELECT comment, comment_id, claim_id, timestamp, is_hidden, parent_id, + channel_name, channel_id, channel_url, signature, signing_ts FROM COMMENTS_ON_CLAIMS """ @@ -164,7 +158,7 @@ def insert_reply(conn: sqlite3.Connection, comment: str, parent_id: str, def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict: with conn: - curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS_CLAIMID + "WHERE comment_id = ?", (comment_id,)) + curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,)) thing = curry.fetchone() return clean(dict(thing)) if thing else None @@ -199,7 +193,7 @@ def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.U placeholders = ', '.join('?' for _ in comment_ids) with conn: return [clean(dict(row)) for row in conn.execute( - SELECT_COMMENTS_ON_CLAIMS_CLAIMID + f'WHERE comment_id IN ({placeholders})', + SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})', tuple(comment_ids) )] From 25eb4f9acdcfedc80259f53bf92f22225d35cdac Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 6 Jan 2020 23:16:46 -0500 Subject: [PATCH 02/46] Prevents claim resolve error from disrupting entire hide operation --- src/database/writes.py | 6 ++++-- src/server/misc.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/database/writes.py b/src/database/writes.py index e724521..0e7b17d 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -88,7 +88,10 @@ async def hide_comments(app, pieces: list) -> list: for p in pieces: claim_id = comment_cids[p['comment_id']] if claim_id not in claims: - claims[claim_id] = await get_claim_from_id(app, claim_id, no_totals=True) + claim = await get_claim_from_id(app, claim_id) + if claim: + claims[claim_id] = claim + channel = claims[claim_id].get('signing_channel') if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']): comments_to_hide.append(p) @@ -100,7 +103,6 @@ async def hide_comments(app, pieces: list) -> list: app, 'UPDATE', db.get_comments_by_id(app['reader'], comment_ids) ) ) - await job.wait() return comment_ids diff --git a/src/server/misc.py b/src/server/misc.py index 593e61a..4f620e6 100644 --- a/src/server/misc.py +++ b/src/server/misc.py @@ -8,7 +8,10 @@ ID_LIST = {'claim_id', 'parent_id', 'comment_id', 'channel_id'} async def get_claim_from_id(app, claim_id, **kwargs): - return (await request_lbrynet(app, 'claim_search', claim_id=claim_id, **kwargs))['items'][0] + try: + return (await request_lbrynet(app, 'claim_search', claim_id=claim_id, **kwargs))['items'][0] + except IndexError: + return def clean_input_params(kwargs: dict): From 6a00c7aa82332f9c4971a19a3cb0306b65c42a4d Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Sun, 19 Jan 2020 23:11:07 -0500 Subject: [PATCH 03/46] Better error logging --- src/server/errors.py | 10 ++++------ src/server/handles.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/server/errors.py b/src/server/errors.py index c0b8af2..273e4bd 100644 --- a/src/server/errors.py +++ b/src/server/errors.py @@ -29,16 +29,14 @@ def make_error(error, exc=None) -> dict: return body -async def report_error(app, exc, msg=''): +async def report_error(app, exc, body: dict): try: if 'slack_webhook' in app['config']: - if msg: - msg = f'"{msg}"' - body = { - "text": f"Got `{type(exc).__name__}`: ```\n{str(exc)}```\n{msg}" + message = { + "text": f"Got `{type(exc).__name__}`: `\n{str(exc)}`\n```{body}```" } async with aiohttp.ClientSession() as sesh: - async with sesh.post(app['config']['slack_webhook'], json=body) as resp: + async with sesh.post(app['config']['slack_webhook'], json=message) as resp: await resp.wait_for_close() except Exception: diff --git a/src/server/handles.py b/src/server/handles.py index 63503f3..6f629a0 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -88,7 +88,7 @@ async def process_json(app, body: dict) -> dict: response['error'] = make_error('INVALID_PARAMS', err) else: response['error'] = make_error('INTERNAL', err) - await app['webhooks'].spawn(report_error(app, err)) + await app['webhooks'].spawn(report_error(app, err, body)) finally: end = time.time() From e9a8a3935c411d71a71b267d2745b7ada4af37f2 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Sun, 19 Jan 2020 23:14:01 -0500 Subject: [PATCH 04/46] Add gitignore --- _trial_temp/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 _trial_temp/.gitignore diff --git a/_trial_temp/.gitignore b/_trial_temp/.gitignore new file mode 100644 index 0000000..c7e09d7 --- /dev/null +++ b/_trial_temp/.gitignore @@ -0,0 +1 @@ +config/conf.json From 6329ef1011b0ad9c71ae2e581de3a9bb50d8255e Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Sun, 19 Jan 2020 23:16:07 -0500 Subject: [PATCH 05/46] add .gitignore --- _trial_temp/.gitignore => .gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename _trial_temp/.gitignore => .gitignore (100%) diff --git a/_trial_temp/.gitignore b/.gitignore similarity index 100% rename from _trial_temp/.gitignore rename to .gitignore From ac69cd6966b59921e15599bc0ae212e403ff24d5 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 18 Feb 2020 14:36:38 -0500 Subject: [PATCH 06/46] Requires credential input for comment creation --- src/database/writes.py | 5 ++--- src/server/validation.py | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/database/writes.py b/src/database/writes.py index 0e7b17d..b9d4b98 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -18,8 +18,7 @@ logger = logging.getLogger(__name__) def create_comment_or_error(conn, comment, claim_id=None, channel_id=None, channel_name=None, signature=None, signing_ts=None, parent_id=None) -> dict: - if channel_id and channel_name: - insert_channel_or_error(conn, channel_name, channel_id) + insert_channel_or_error(conn, channel_name, channel_id) fn = db.insert_comment if parent_id is None else db.insert_reply comment_id = fn( conn=conn, @@ -65,7 +64,7 @@ async def _abandon_comment(app, comment_id): # DELETE async def create_comment(app, params): - if is_valid_base_comment(**params) and is_valid_credential_input(**params): + if is_valid_base_comment(**params): job = await app['comment_scheduler'].spawn(_create_comment(app, params)) comment = await job.wait() if comment: diff --git a/src/server/validation.py b/src/server/validation.py index ad99f0c..43f05f6 100644 --- a/src/server/validation.py +++ b/src/server/validation.py @@ -51,23 +51,25 @@ def claim_id_is_valid(claim_id: str) -> bool: def is_valid_base_comment(comment: str, claim_id: str, parent_id: str = None, **kwargs) -> bool: - return comment is not None and body_is_valid(comment) and \ - ((claim_id is not None and claim_id_is_valid(claim_id)) or - (parent_id is not None and comment_id_is_valid(parent_id))) + return comment and body_is_valid(comment) and \ + ((claim_id and claim_id_is_valid(claim_id)) or # parentid is used in place of claimid in replies + (parent_id and comment_id_is_valid(parent_id))) \ + and is_valid_credential_input(**kwargs) def is_valid_credential_input(channel_id: str = None, channel_name: str = None, signature: str = None, signing_ts: str = None, **kwargs) -> bool: - if channel_id or channel_name or signature or signing_ts: - try: - assert channel_id and channel_name and signature and signing_ts - assert is_valid_channel(channel_id, channel_name) - assert len(signature) == 128 - assert signing_ts.isalnum() - - except Exception: - return False - return True + try: + assert channel_id and channel_name and signature and signing_ts + assert is_valid_channel(channel_id, channel_name) + assert len(signature) == 128 + assert signing_ts.isalnum() + except Exception as e: + logger.exception(f'Failed to validate channel: lbry://{channel_name}#{channel_id}, ' + f'signature: {signature} signing_ts: {signing_ts}') + return False + finally: + return True def validate_signature_from_claim(claim: dict, signature: typing.Union[str, bytes], From 723026f967cab6b24f2431f9183a17e0030100af Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 18 Feb 2020 15:26:04 -0500 Subject: [PATCH 07/46] Proper kwarg management --- src/server/validation.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/server/validation.py b/src/server/validation.py index 43f05f6..ef1c841 100644 --- a/src/server/validation.py +++ b/src/server/validation.py @@ -50,7 +50,8 @@ def claim_id_is_valid(claim_id: str) -> bool: return re.fullmatch('([a-z0-9]{40}|[A-Z0-9]{40})', claim_id) is not None -def is_valid_base_comment(comment: str, claim_id: str, parent_id: str = None, **kwargs) -> bool: +# default to None so params can be treated as kwargs; param count becomes more manageable +def is_valid_base_comment(comment: str = None, claim_id: str = None, parent_id: str = None, **kwargs) -> bool: return comment and body_is_valid(comment) and \ ((claim_id and claim_id_is_valid(claim_id)) or # parentid is used in place of claimid in replies (parent_id and comment_id_is_valid(parent_id))) \ @@ -58,18 +59,19 @@ def is_valid_base_comment(comment: str, claim_id: str, parent_id: str = None, ** def is_valid_credential_input(channel_id: str = None, channel_name: str = None, - signature: str = None, signing_ts: str = None, **kwargs) -> bool: + signature: str = None, signing_ts: str = None) -> bool: try: - assert channel_id and channel_name and signature and signing_ts + assert None not in (channel_id, channel_name, signature, signing_ts) assert is_valid_channel(channel_id, channel_name) assert len(signature) == 128 assert signing_ts.isalnum() + + return True + except Exception as e: logger.exception(f'Failed to validate channel: lbry://{channel_name}#{channel_id}, ' f'signature: {signature} signing_ts: {signing_ts}') return False - finally: - return True def validate_signature_from_claim(claim: dict, signature: typing.Union[str, bytes], From 220ceefbd242c63d67dee916714ca567c2c790f1 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 18 Feb 2020 15:27:29 -0500 Subject: [PATCH 08/46] Uses is_valid_base_comment instead of static proxy method --- test/test_server.py | 63 ++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/test/test_server.py b/test/test_server.py index 3c659d9..5cd9ea6 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -11,7 +11,6 @@ from faker.providers import misc from src.settings import config from src.server import app -from src.server.validation import is_valid_channel from src.server.validation import is_valid_base_comment from test.testcase import AsyncioTestCase @@ -97,22 +96,6 @@ class ServerTest(AsyncioTestCase): async def post_comment(self, **params): return await jsonrpc_post(self.url, 'create_comment', **params) - @staticmethod - def is_valid_message(comment=None, claim_id=None, parent_id=None, - channel_name=None, channel_id=None, signature=None, signing_ts=None): - try: - assert is_valid_base_comment(comment, claim_id, parent_id) - - if channel_name or channel_id or signature or signing_ts: - assert channel_id and channel_name and signature and signing_ts - assert is_valid_channel(channel_id, channel_name) - assert len(signature) == 128 - assert signing_ts.isalnum() - - except Exception: - return False - return True - async def test01CreateCommentNoReply(self): anonymous_test = create_test_comments( ('claim_id', 'channel_id', 'channel_name', 'comment'), @@ -122,13 +105,13 @@ class ServerTest(AsyncioTestCase): claim_id=None ) for test in anonymous_test: - with self.subTest(test=test): + with self.subTest(test='null fields: ' + ', '.join(k for k, v in test.items() if not v)): message = await self.post_comment(**test) self.assertTrue('result' in message or 'error' in message) if 'error' in message: - self.assertFalse(self.is_valid_message(**test)) + self.assertFalse(is_valid_base_comment(**test)) else: - self.assertTrue(self.is_valid_message(**test)) + self.assertTrue(is_valid_base_comment(**test)) async def test02CreateNamedCommentsNoReply(self): named_test = create_test_comments( @@ -144,9 +127,9 @@ class ServerTest(AsyncioTestCase): message = await self.post_comment(**test) self.assertTrue('result' in message or 'error' in message) if 'error' in message: - self.assertFalse(self.is_valid_message(**test)) + self.assertFalse(is_valid_base_comment(**test)) else: - self.assertTrue(self.is_valid_message(**test)) + self.assertTrue(is_valid_base_comment(**test)) async def test03CreateAllTestComments(self): test_all = create_test_comments(replace.keys(), **{ @@ -157,9 +140,9 @@ class ServerTest(AsyncioTestCase): message = await self.post_comment(**test) self.assertTrue('result' in message or 'error' in message) if 'error' in message: - self.assertFalse(self.is_valid_message(**test)) + self.assertFalse(is_valid_base_comment(**test)) else: - self.assertTrue(self.is_valid_message(**test)) + self.assertTrue(is_valid_base_comment(**test)) async def test04CreateAllReplies(self): claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8' @@ -189,9 +172,37 @@ class ServerTest(AsyncioTestCase): message = await self.post_comment(**test) self.assertTrue('result' in message or 'error' in message) if 'error' in message: - self.assertFalse(self.is_valid_message(**test)) + self.assertFalse(is_valid_base_comment(**test)) else: - self.assertTrue(self.is_valid_message(**test)) + self.assertTrue(is_valid_base_comment(**test)) + + async def testSlackWebhook(self): + claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8' + channel_name = '@name' + channel_id = fake.sha1() + signature = '{}'*64 + signing_ts = '1234' + + base = await self.post_comment( + channel_name=channel_name, + channel_id=channel_id, + comment='duplicate', + claim_id=claim_id, + signing_ts=signing_ts, + signature=signature + ) + + comment_id = base['result']['comment_id'] + + with self.subTest(test=comment_id): + await self.post_comment( + channel_name=channel_name, + channel_id=channel_id, + comment='duplicate', + claim_id=claim_id, + signing_ts=signing_ts, + signature=signature + ) class ListCommentsTest(AsyncioTestCase): From a825c6a4b9cadc45f150d0bd1f7e8c8e750a62cf Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 18 Feb 2020 15:28:46 -0500 Subject: [PATCH 09/46] Anonymous comment unit-test now tests against anonymous comments --- test/test_database.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/test_database.py b/test/test_database.py index 347494d..bd10d16 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -1,4 +1,4 @@ -import unittest +import sqlite3 from random import randint import faker @@ -53,21 +53,13 @@ class TestDatabaseOperations(DatabaseTestCase): self.assertEqual(reply['parent_id'], comment['comment_id']) def test02AnonymousComments(self): - comment = create_comment_or_error( + self.assertRaises( + sqlite3.IntegrityError, + create_comment_or_error, conn=self.conn, claim_id=self.claimId, comment='This is an ANONYMOUS comment' ) - self.assertIsNotNone(comment) - previous_id = comment['comment_id'] - reply = create_comment_or_error( - conn=self.conn, - claim_id=self.claimId, - comment='This is an unnamed response', - parent_id=previous_id - ) - self.assertIsNotNone(reply) - self.assertEqual(reply['parent_id'], comment['comment_id']) def test03SignedComments(self): comment = create_comment_or_error( From 77d499a0a3c30722f9b69fe63438a08b382b285e Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 18 Feb 2020 15:28:59 -0500 Subject: [PATCH 10/46] Removes unused tests --- test/test_database.py | 58 ++----------------------------------------- 1 file changed, 2 insertions(+), 56 deletions(-) diff --git a/test/test_database.py b/test/test_database.py index bd10d16..c43bb0a 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -134,61 +134,7 @@ class TestDatabaseOperations(DatabaseTestCase): comment='this username is too short' ) - def test05InsertRandomComments(self): - # TODO: Fix this test into something practical - self.skipTest('This is a bad test') - top_comments, claim_ids = generate_top_comments_random() - total = 0 - success = 0 - for _, comments in top_comments.items(): - for i, comment in enumerate(comments): - with self.subTest(comment=comment): - result = create_comment_or_error(self.conn, **comment) - if result: - success += 1 - comments[i] = result - del comment - total += len(comments) - self.assertLessEqual(success, total) - self.assertGreater(success, 0) - success = 0 - for reply in generate_replies_random(top_comments): - reply_id = create_comment_or_error(self.conn, **reply) - if reply_id: - success += 1 - self.assertGreater(success, 0) - self.assertLess(success, total) - del top_comments - del claim_ids - - def test06GenerateAndListComments(self): - # TODO: Make this test not suck - self.skipTest('this is a stupid test') - top_comments, claim_ids = generate_top_comments() - total, success = 0, 0 - for _, comments in top_comments.items(): - for i, comment in enumerate(comments): - result = create_comment_or_error(self.conn, **comment) - if result: - success += 1 - comments[i] = result - del comment - total += len(comments) - self.assertEqual(total, success) - self.assertGreater(total, 0) - for reply in generate_replies(top_comments): - create_comment_or_error(self.conn, **reply) - for claim_id in claim_ids: - comments_ids = get_comment_ids(self.conn, claim_id) - with self.subTest(comments_ids=comments_ids): - self.assertIs(type(comments_ids), list) - self.assertGreaterEqual(len(comments_ids), 0) - self.assertLessEqual(len(comments_ids), 50) - replies = get_comments_by_id(self.conn, comments_ids) - self.assertLessEqual(len(replies), 50) - self.assertEqual(len(replies), len(comments_ids)) - - def test07HideComments(self): + def test05HideComments(self): comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() self.assertFalse(comment['is_hidden']) @@ -201,7 +147,7 @@ class TestDatabaseOperations(DatabaseTestCase): comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() self.assertTrue(comment['is_hidden']) - def test08DeleteComments(self): + def test06DeleteComments(self): comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') comments = get_claim_comments(self.conn, self.claimId) match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items'])) From 0529fa7d015f959186380ce4ad5ce19ea9b0a7f0 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 27 Mar 2020 01:24:34 -0400 Subject: [PATCH 11/46] Implements DB model and preliminary select queries --- src/database/ddl.py | 159 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 src/database/ddl.py diff --git a/src/database/ddl.py b/src/database/ddl.py new file mode 100644 index 0000000..6593cd0 --- /dev/null +++ b/src/database/ddl.py @@ -0,0 +1,159 @@ +import json +import logging + +import math +import timeit + +import typing + +from peewee import ModelSelect +from playhouse.shortcuts import model_to_dict +from peewee import * + + +def get_database_connection(): + # for now it's an sqlite database + db = SqliteDatabase() + return db + + +database = get_database_connection() + + +class BaseModel(Model): + class Meta: + database = database + + +class Channel(BaseModel): + claim_id = TextField(column_name='ClaimId', primary_key=True) + name = TextField(column_name='Name') + + class Meta: + table_name = 'CHANNEL' + + +class Comment(BaseModel): + comment = TextField(column_name='Body') + channel = ForeignKeyField( + backref='comments', + column_name='ChannelId', + field='claim_id', + model=Channel, + null=True + ) + comment_id = TextField(column_name='CommentId', primary_key=True) + is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT FALSE")]) + claim_id = TextField(column_name='LbryClaimId') + parent = ForeignKeyField( + column_name='ParentId', + field='comment_id', + model='self', + null=True, + backref='replies' + ) + signature = TextField(column_name='Signature', null=True, unique=True) + signing_ts = TextField(column_name='SigningTs', null=True) + timestamp = IntegerField(column_name='Timestamp') + + class Meta: + table_name = 'COMMENT' + indexes = ( + (('author', 'comment_id'), False), + (('claim_id', 'comment_id'), False), + ) + + +COMMENT_FIELDS = [ + Comment.comment, + Comment.comment_id, + Comment.claim_id, + Comment.timestamp, + Comment.signature, + Comment.signing_ts, + Comment.is_hidden, + Comment.parent.alias('parent_id'), +] + +CHANNEL_FIELDS = [ + Channel.claim_id.alias('channel_id'), + Channel.name.alias('channel_name') +] + + +def get_comment_list(claim_id: str = None, parent_id: str = None, + top_level: bool = False, exclude_mode: str = None, + page: int = 1, page_size: int = 50, expressions=None) -> dict: + query = Comment.select(*COMMENT_FIELDS, *CHANNEL_FIELDS) + if claim_id: + query = query.where(Comment.claim_id == claim_id) + if top_level: + query = query.where(Comment.parent.is_null()) + + if parent_id: + query = query.where(Comment.ParentId == parent_id) + + if exclude_mode: + show_hidden = exclude_mode.lower() == 'hidden' + query = query.where((Comment.is_hidden == show_hidden)) + total = query.count() + query = (query + .join(Channel, JOIN.LEFT_OUTER) + .where(expressions) + .order_by(Comment.timestamp.desc()) + .paginate(page, page_size)) + items = [clean(item) for item in query.dicts()] + # has_hidden_comments is deprecated + data = { + 'page': page, + 'page_size': page_size, + 'total_pages': math.ceil(total / page_size), + 'total_items': total, + 'items': items, + 'has_hidden_comments': exclude_mode is not None and exclude_mode == 'hidden', + } + return data + + +def clean(thing: dict) -> dict: + return {k: v for k, v in thing.items() if v is not None} + + +def get_comment(comment_id: str) -> dict: + try: + comment: Comment = Comment.get_by_id(comment_id) + except DoesNotExist as e: + raise ValueError from e + else: + as_dict = model_to_dict(comment) + if comment.channel: + as_dict.update({ + 'channel_id': comment.channel_id, + 'channel_name': comment.channel.name, + 'signature': comment.signature, + 'signing_ts': comment.signing_ts, + 'channel_url': f'lbry://{comment.channel.name}#{comment.channel_id}' + }) + if comment.parent: + as_dict.update({ + 'parent_id': comment.parent_id + }) + return clean(as_dict) + + +if __name__ == '__main__': + logger = logging.getLogger('peewee') + logger.addHandler(logging.StreamHandler()) + logger.setLevel(logging.DEBUG) + + comment_list = get_comment_list( + page_size=1, + expressions=(Comment.channel.is_null()) + ) + + comment = comment_list['items'].pop() + print(json.dumps(comment, indent=4)) + other_comment = get_comment(comment['comment_id']) + + print(json.dumps(other_comment, indent=4)) + print(comment == other_comment) From a581425a64988da11377ae964fc4180fdb5adf0a Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 27 Mar 2020 01:26:13 -0400 Subject: [PATCH 12/46] Shifts from JSON configuration to yml based --- setup.py | 3 +++ src/definitions.py | 7 +++++++ src/main.py | 42 +++++++++++++++++++++++++++++++++++------- src/server/app.py | 9 +++------ src/settings.py | 17 ----------------- test/test_server.py | 5 ++++- 6 files changed, 52 insertions(+), 31 deletions(-) create mode 100644 src/definitions.py delete mode 100644 src/settings.py diff --git a/setup.py b/setup.py index 923e82a..25087e4 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ setup( data_files=[('config', ['config/conf.json',])], include_package_data=True, install_requires=[ + 'mysql-connector-python', + 'pyyaml', 'Faker>=1.0.7', 'asyncio>=3.4.3', 'aiohttp==3.5.4', @@ -24,5 +26,6 @@ setup( 'PyNaCl>=1.3.0', 'requests', 'cython', + 'peewee' ] ) diff --git a/src/definitions.py b/src/definitions.py new file mode 100644 index 0000000..9972e13 --- /dev/null +++ b/src/definitions.py @@ -0,0 +1,7 @@ +import os + +SRC_DIR = os.path.dirname(os.path.abspath(__file__)) +ROOT_DIR = os.path.dirname(SRC_DIR) +CONFIG_FILE = os.path.join(ROOT_DIR, 'config', 'conf.json') +LOGGING_DIR = os.path.join(ROOT_DIR, 'logs') +DATABASE_DIR = os.path.join(ROOT_DIR, 'database') diff --git a/src/main.py b/src/main.py index b31bcea..c22a82b 100644 --- a/src/main.py +++ b/src/main.py @@ -1,13 +1,20 @@ import argparse +import json +import yaml import logging import logging.config +import os import sys from src.server.app import run_app -from src.settings import config +from src.definitions import LOGGING_DIR, CONFIG_FILE, DATABASE_DIR -def config_logging_from_settings(conf): +def setup_logging_from_config(conf: dict): + # set the logging directory here from the settings file + if not os.path.exists(LOGGING_DIR): + os.mkdir(LOGGING_DIR) + _config = { "version": 1, "disable_existing_loggers": False, @@ -32,7 +39,7 @@ def config_logging_from_settings(conf): "level": "DEBUG", "formatter": "standard", "class": "logging.handlers.RotatingFileHandler", - "filename": conf['path']['debug_log'], + "filename": os.path.join(LOGGING_DIR, 'debug.log'), "maxBytes": 10485760, "backupCount": 5 }, @@ -40,7 +47,7 @@ def config_logging_from_settings(conf): "level": "ERROR", "formatter": "standard", "class": "logging.handlers.RotatingFileHandler", - "filename": conf['path']['error_log'], + "filename": os.path.join(LOGGING_DIR, 'error.log'), "maxBytes": 10485760, "backupCount": 5 }, @@ -48,7 +55,7 @@ def config_logging_from_settings(conf): "level": "NOTSET", "formatter": "aiohttp", "class": "logging.handlers.RotatingFileHandler", - "filename": conf['path']['server_log'], + "filename": os.path.join(LOGGING_DIR, 'server.log'), "maxBytes": 10485760, "backupCount": 5 } @@ -70,15 +77,36 @@ def config_logging_from_settings(conf): logging.config.dictConfig(_config) +def get_config(filepath): + with open(filepath, 'r') as cfile: + config = yaml.load(cfile, Loader=yaml.FullLoader) + return config + + +def setup_db_from_config(config: dict): + if 'sqlite' in config['database']: + if not os.path.exists(DATABASE_DIR): + os.mkdir(DATABASE_DIR) + + config['db_path'] = os.path.join( + DATABASE_DIR, config['database']['sqlite'] + ) + + def main(argv=None): argv = argv or sys.argv[1:] parser = argparse.ArgumentParser(description='LBRY Comment Server') parser.add_argument('--port', type=int) + parser.add_argument('--config', type=str) args = parser.parse_args(argv) - config_logging_from_settings(config) + + config = get_config(CONFIG_FILE) if not args.config else args.config + setup_logging_from_config(config) + setup_db_from_config(config) + if args.port: config['port'] = args.port - config_logging_from_settings(config) + run_app(config) diff --git a/src/server/app.py b/src/server/app.py index a25787b..446c378 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -75,12 +75,9 @@ class CommentDaemon: self.config = app['config'] # configure the db file - if db_file: - app['db_path'] = db_file - app['backup'] = backup - else: - app['db_path'] = config['path']['database'] - app['backup'] = backup or (app['db_path'] + '.backup') + app['db_path'] = db_file or config.get('db_path') + if app['db_path']: + app['backup'] = backup or '.'.join((app['db_path'], 'backup')) # configure the order of tasks to run during app lifetime app.on_startup.append(setup_db_schema) diff --git a/src/settings.py b/src/settings.py deleted file mode 100644 index 2d720c2..0000000 --- a/src/settings.py +++ /dev/null @@ -1,17 +0,0 @@ -# cython: language_level=3 -import json -import pathlib - -root_dir = pathlib.Path(__file__).parent.parent -config_path = root_dir / 'config' / 'conf.json' - - -def get_config(filepath): - with open(filepath, 'r') as cfile: - conf = json.load(cfile) - for key, path in conf['path'].items(): - conf['path'][key] = str(root_dir / path) - return conf - - -config = get_config(config_path) diff --git a/test/test_server.py b/test/test_server.py index 5cd9ea6..55fcdba 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -9,13 +9,16 @@ from faker.providers import internet from faker.providers import lorem from faker.providers import misc -from src.settings import config +from src.main import get_config, CONFIG_FILE from src.server import app from src.server.validation import is_valid_base_comment from test.testcase import AsyncioTestCase +config = get_config(CONFIG_FILE) + + if 'slack_webhook' in config: config.pop('slack_webhook') From cc20088b06e90a92b4f529f17f1c6b7e3e8d1171 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 27 Mar 2020 05:07:56 -0400 Subject: [PATCH 13/46] Implements all search queries --- src/database/ddl.py | 104 +++++++++++++++++++++++++++++----------- src/database/queries.py | 2 + src/server/handles.py | 16 +++---- 3 files changed, 85 insertions(+), 37 deletions(-) diff --git a/src/database/ddl.py b/src/database/ddl.py index 6593cd0..51d2de8 100644 --- a/src/database/ddl.py +++ b/src/database/ddl.py @@ -64,27 +64,34 @@ class Comment(BaseModel): ) -COMMENT_FIELDS = [ - Comment.comment, - Comment.comment_id, - Comment.claim_id, - Comment.timestamp, - Comment.signature, - Comment.signing_ts, - Comment.is_hidden, - Comment.parent.alias('parent_id'), -] - -CHANNEL_FIELDS = [ - Channel.claim_id.alias('channel_id'), - Channel.name.alias('channel_name') -] +FIELDS = { + 'comment': Comment.comment, + 'comment_id': Comment.comment_id, + 'claim_id': Comment.claim_id, + 'timestamp': Comment.timestamp, + 'signature': Comment.signature, + 'signing_ts': Comment.signing_ts, + 'is_hidden': Comment.is_hidden, + 'parent_id': Comment.parent.alias('parent_id'), + 'channel_id': Channel.claim_id.alias('channel_id'), + 'channel_name': Channel.name.alias('channel_name'), + 'channel_url': ('lbry://' + Channel.name + '#' + Channel.claim_id).alias('channel_url') +} -def get_comment_list(claim_id: str = None, parent_id: str = None, - top_level: bool = False, exclude_mode: str = None, - page: int = 1, page_size: int = 50, expressions=None) -> dict: - query = Comment.select(*COMMENT_FIELDS, *CHANNEL_FIELDS) +def comment_list(claim_id: str = None, parent_id: str = None, + top_level: bool = False, exclude_mode: str = None, + page: int = 1, page_size: int = 50, expressions=None, + select_fields: list = None, exclude_fields: list = None) -> dict: + fields = FIELDS.keys() + if exclude_fields: + fields -= set(exclude_fields) + if select_fields: + fields &= set(select_fields) + attributes = [FIELDS[field] for field in fields] + query = Comment.select(*attributes) + + # todo: allow this process to be more automated, so it can just be an expression if claim_id: query = query.where(Comment.claim_id == claim_id) if top_level: @@ -96,10 +103,13 @@ def get_comment_list(claim_id: str = None, parent_id: str = None, if exclude_mode: show_hidden = exclude_mode.lower() == 'hidden' query = query.where((Comment.is_hidden == show_hidden)) + + if expressions: + query = query.where(expressions) + total = query.count() query = (query .join(Channel, JOIN.LEFT_OUTER) - .where(expressions) .order_by(Comment.timestamp.desc()) .paginate(page, page_size)) items = [clean(item) for item in query.dicts()] @@ -132,7 +142,7 @@ def get_comment(comment_id: str) -> dict: 'channel_name': comment.channel.name, 'signature': comment.signature, 'signing_ts': comment.signing_ts, - 'channel_url': f'lbry://{comment.channel.name}#{comment.channel_id}' + 'channel_url': comment.channel.channel_url }) if comment.parent: as_dict.update({ @@ -141,19 +151,55 @@ def get_comment(comment_id: str) -> dict: return clean(as_dict) +def get_comment_ids(claim_id: str = None, parent_id: str = None, + page: int = 1, page_size: int = 50, flattened=False) -> dict: + results = comment_list( + claim_id, parent_id, + top_level=(parent_id is None), + page=page, page_size=page_size, + select_fields=['comment_id', 'parent_id'] + ) + if flattened: + results.update({ + 'items': [item['comment_id'] for item in results['items']], + 'replies': [(item['comment_id'], item.get('parent_id')) for item in results['items']] + }) + return results + + +def get_comments_by_id(comment_ids: typing.Union[list, tuple]) -> dict: + expression = Comment.comment_id.in_(comment_ids) + return comment_list(expressions=expression, page_size=len(comment_ids)) + + +def get_channel_from_comment_id(comment_id: str) -> dict: + try: + comment = Comment.get_by_id(comment_id) + except DoesNotExist as e: + raise ValueError from e + else: + channel = comment.channel + if not channel: + raise ValueError('The provided comment does not belong to a channel.') + return { + 'channel_name': channel.name, + 'channel_id': channel.claim_id, + 'channel_url': 'lbry://' + channel.name + '#' + channel.claim_id + } + + if __name__ == '__main__': logger = logging.getLogger('peewee') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) - comment_list = get_comment_list( - page_size=1, - expressions=(Comment.channel.is_null()) + comments = comment_list( + page_size=20, + expressions=((Comment.timestamp < 1583272089) & + (Comment.claim_id ** '420%')) ) - comment = comment_list['items'].pop() - print(json.dumps(comment, indent=4)) - other_comment = get_comment(comment['comment_id']) + ids = get_comment_ids('4207d2378bf4340e68c9d88faf7ee24ea1a1f95a') - print(json.dumps(other_comment, indent=4)) - print(comment == other_comment) + print(json.dumps(comments, indent=4)) + print(json.dumps(ids, indent=4)) \ No newline at end of file diff --git a/src/database/queries.py b/src/database/queries.py index 03815aa..662c22f 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -35,6 +35,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = page: int = 1, page_size: int = 50, top_level=False): with conn: if top_level: + # doesn't include any results = [clean(dict(row)) for row in conn.execute( SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?", (claim_id, page_size, page_size * (page - 1)) @@ -44,6 +45,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = (claim_id,) ) elif parent_id is None: + # include all, no specific parent comment results = [clean(dict(row)) for row in conn.execute( SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ", (claim_id, page_size, page_size * (page - 1)) diff --git a/src/server/handles.py b/src/server/handles.py index 6f629a0..c2d5f3e 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -55,16 +55,16 @@ async def handle_edit_comment(app, params): METHODS = { 'ping': ping, - 'get_claim_comments': handle_get_claim_comments, - 'get_claim_hidden_comments': handle_get_claim_hidden_comments, + 'get_claim_comments': handle_get_claim_comments, # this gets used + 'get_claim_hidden_comments': handle_get_claim_hidden_comments, # this gets used 'get_comment_ids': handle_get_comment_ids, - 'get_comments_by_id': handle_get_comments_by_id, - 'get_channel_from_comment_id': handle_get_channel_from_comment_id, - 'create_comment': create_comment, + 'get_comments_by_id': handle_get_comments_by_id, # this gets used + 'get_channel_from_comment_id': handle_get_channel_from_comment_id, # this gets used + 'create_comment': create_comment, # this gets used 'delete_comment': handle_abandon_comment, - 'abandon_comment': handle_abandon_comment, - 'hide_comments': handle_hide_comments, - 'edit_comment': handle_edit_comment + 'abandon_comment': handle_abandon_comment, # this gets used + 'hide_comments': handle_hide_comments, # this gets used + 'edit_comment': handle_edit_comment # this gets used } From 644e5e84771fc521744f60b3190c378a3e22f7b9 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 27 Mar 2020 15:59:57 -0400 Subject: [PATCH 14/46] adds get_channel_from_id & simplifies to all call comment_list --- src/database/ddl.py | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/src/database/ddl.py b/src/database/ddl.py index 51d2de8..491d521 100644 --- a/src/database/ddl.py +++ b/src/database/ddl.py @@ -130,25 +130,9 @@ def clean(thing: dict) -> dict: def get_comment(comment_id: str) -> dict: - try: - comment: Comment = Comment.get_by_id(comment_id) - except DoesNotExist as e: - raise ValueError from e - else: - as_dict = model_to_dict(comment) - if comment.channel: - as_dict.update({ - 'channel_id': comment.channel_id, - 'channel_name': comment.channel.name, - 'signature': comment.signature, - 'signing_ts': comment.signing_ts, - 'channel_url': comment.channel.channel_url - }) - if comment.parent: - as_dict.update({ - 'parent_id': comment.parent_id - }) - return clean(as_dict) + return (comment_list(expressions=(Comment.comment_id == comment_id), page_size=1) + .get('items') + .pop()) def get_comment_ids(claim_id: str = None, parent_id: str = None, @@ -173,19 +157,12 @@ def get_comments_by_id(comment_ids: typing.Union[list, tuple]) -> dict: def get_channel_from_comment_id(comment_id: str) -> dict: - try: - comment = Comment.get_by_id(comment_id) - except DoesNotExist as e: - raise ValueError from e - else: - channel = comment.channel - if not channel: - raise ValueError('The provided comment does not belong to a channel.') - return { - 'channel_name': channel.name, - 'channel_id': channel.claim_id, - 'channel_url': 'lbry://' + channel.name + '#' + channel.claim_id - } + results = comment_list( + expressions=(Comment.comment_id == comment_id), + select_fields=['channel_name', 'channel_id', 'channel_url'], + page_size=1 + ) + return results['items'].pop() if __name__ == '__main__': From 63f2c7e9e06a6d08d4c33d155584c08b2486a2f4 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 27 Mar 2020 17:44:22 -0400 Subject: [PATCH 15/46] Moves the clean method to misc.py, moves misc.py --- src/database/ddl.py | 15 ++++++--------- src/database/writes.py | 2 +- src/{server => }/misc.py | 4 ++++ src/server/handles.py | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) rename src/{server => }/misc.py (86%) diff --git a/src/database/ddl.py b/src/database/ddl.py index 491d521..4a0d9c3 100644 --- a/src/database/ddl.py +++ b/src/database/ddl.py @@ -1,14 +1,15 @@ import json +import time + import logging - import math -import timeit - import typing -from peewee import ModelSelect -from playhouse.shortcuts import model_to_dict from peewee import * +import nacl.hash + +from src.server.validation import is_valid_base_comment +from src.misc import clean def get_database_connection(): @@ -125,10 +126,6 @@ def comment_list(claim_id: str = None, parent_id: str = None, return data -def clean(thing: dict) -> dict: - return {k: v for k, v in thing.items() if v is not None} - - def get_comment(comment_id: str) -> dict: return (comment_list(expressions=(Comment.comment_id == comment_id), page_size=1) .get('items') diff --git a/src/database/writes.py b/src/database/writes.py index b9d4b98..9f13f3f 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -7,7 +7,7 @@ from src.server.validation import is_valid_base_comment from src.server.validation import is_valid_credential_input from src.server.validation import validate_signature_from_claim from src.server.validation import body_is_valid -from src.server.misc import get_claim_from_id +from src.misc import get_claim_from_id from src.server.external import send_notifications from src.server.external import send_notification import src.database.queries as db diff --git a/src/server/misc.py b/src/misc.py similarity index 86% rename from src/server/misc.py rename to src/misc.py index 4f620e6..e9d5be8 100644 --- a/src/server/misc.py +++ b/src/misc.py @@ -20,3 +20,7 @@ def clean_input_params(kwargs: dict): kwargs[k] = v.strip() if k in ID_LIST: kwargs[k] = v.lower() + + +def clean(thing: dict) -> dict: + return {k: v for k, v in thing.items() if v is not None} \ No newline at end of file diff --git a/src/server/handles.py b/src/server/handles.py index c2d5f3e..a63504b 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -9,7 +9,7 @@ import src.database.queries as db from src.database.writes import abandon_comment, create_comment from src.database.writes import hide_comments from src.database.writes import edit_comment -from src.server.misc import clean_input_params +from src.misc import clean_input_params from src.server.errors import make_error, report_error logger = logging.getLogger(__name__) From 510f2a5d29ca2dd71a960fc69472e0228d7daf96 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 27 Mar 2020 17:44:51 -0400 Subject: [PATCH 16/46] Adds comment create logic --- src/database/ddl.py | 53 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/database/ddl.py b/src/database/ddl.py index 4a0d9c3..7c3c145 100644 --- a/src/database/ddl.py +++ b/src/database/ddl.py @@ -159,9 +159,60 @@ def get_channel_from_comment_id(comment_id: str) -> dict: select_fields=['channel_name', 'channel_id', 'channel_url'], page_size=1 ) + # todo: make the return type here consistent return results['items'].pop() +def create_comment_id(comment: str, channel_id: str, timestamp: int): + # We convert the timestamp from seconds into minutes + # to prevent spammers from commenting the same BS everywhere. + nearest_minute = str(math.floor(timestamp)) + + # don't use claim_id for the comment_id anymore so comments + # are not unique to just one claim + prehash = b':'.join([ + comment.encode(), + channel_id.encode(), + nearest_minute.encode() + ]) + return nacl.hash.sha256(prehash).decode() + + +def create_comment(comment: str = None, claim_id: str = None, + parent_id: str = None, channel_id: str = None, + channel_name: str = None, signature: str = None, + signing_ts: str = None) -> dict: + if not is_valid_base_comment( + comment=comment, + claim_id=claim_id, + parent_id=parent_id, + channel_id=channel_id, + channel_name=channel_name, + signature=signature, + signing_ts=signing_ts + ): + raise ValueError('Invalid Parameters given for comment') + + channel, _ = Channel.get_or_create(name=channel_name, claim_id=channel_id) + if parent_id: + parent: Comment = Comment.get_by_id(parent_id) + claim_id = parent.claim_id + + timestamp = int(time.time()) + comment_id = create_comment_id(comment, channel_id, timestamp) + new_comment = Comment.create( + claim_id=claim_id, + comment_id=comment_id, + comment=comment, + parent=parent_id, + channel=channel, + signature=signature, + signing_ts=signing_ts, + timestamp=timestamp + ) + return get_comment(new_comment.comment_id) + + if __name__ == '__main__': logger = logging.getLogger('peewee') logger.addHandler(logging.StreamHandler()) @@ -176,4 +227,4 @@ if __name__ == '__main__': ids = get_comment_ids('4207d2378bf4340e68c9d88faf7ee24ea1a1f95a') print(json.dumps(comments, indent=4)) - print(json.dumps(ids, indent=4)) \ No newline at end of file + print(json.dumps(ids, indent=4)) From 8138e7166861c7cff5e2786fdda381d82b1e50b1 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 30 Mar 2020 18:02:20 -0400 Subject: [PATCH 17/46] Sets database dynamically --- src/database/ddl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/database/ddl.py b/src/database/ddl.py index 7c3c145..5778965 100644 --- a/src/database/ddl.py +++ b/src/database/ddl.py @@ -18,12 +18,15 @@ def get_database_connection(): return db + +database_proxy = DatabaseProxy() database = get_database_connection() +database_proxy.initialize(database) class BaseModel(Model): class Meta: - database = database + database = database_proxy class Channel(BaseModel): From a22b4a9162454fec9fa3578eb5d0d6badf0ac0f3 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 30 Mar 2020 18:04:20 -0400 Subject: [PATCH 18/46] Adds delete, edit, and hide operations --- src/database/ddl.py | 58 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/src/database/ddl.py b/src/database/ddl.py index 5778965..0983b74 100644 --- a/src/database/ddl.py +++ b/src/database/ddl.py @@ -203,17 +203,53 @@ def create_comment(comment: str = None, claim_id: str = None, timestamp = int(time.time()) comment_id = create_comment_id(comment, channel_id, timestamp) - new_comment = Comment.create( - claim_id=claim_id, - comment_id=comment_id, - comment=comment, - parent=parent_id, - channel=channel, - signature=signature, - signing_ts=signing_ts, - timestamp=timestamp - ) - return get_comment(new_comment.comment_id) + with database_proxy.atomic(): + new_comment = Comment.create( + claim_id=claim_id, + comment_id=comment_id, + comment=comment, + parent=parent_id, + channel=channel, + signature=signature, + signing_ts=signing_ts, + timestamp=timestamp + ) + return get_comment(new_comment.comment_id) + + +def delete_comment(comment_id: str) -> bool: + try: + comment: Comment = Comment.get_by_id(comment_id) + except DoesNotExist as e: + raise ValueError from e + else: + with database_proxy.atomic(): + return 0 < comment.delete_instance(True, delete_nullable=True) + + +def edit_comment(comment_id: str, new_comment: str, new_sig: str, new_ts: str) -> bool: + try: + comment: Comment = Comment.get_by_id(comment_id) + except DoesNotExist as e: + raise ValueError from e + else: + with database_proxy.atomic(): + comment.comment = new_comment + comment.signature = new_sig + comment.signing_ts = new_ts + + # todo: add a 'last-modified' timestamp + comment.timestamp = int(time.time()) + return comment.save() > 0 + + +def set_hidden_flag(comment_ids: typing.List[str], hidden=True) -> bool: + # sets `is_hidden` flag for all `comment_ids` to the `hidden` param + with database_proxy.atomic(): + update = (Comment + .update(is_hidden=hidden) + .where(Comment.comment_id.in_(comment_ids))) + return update.execute() > 0 if __name__ == '__main__': From aee12eba54bc11c0f8323782be428e9662c3d820 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 30 Mar 2020 21:58:07 -0400 Subject: [PATCH 19/46] removes inherited base database in favor of binding api --- src/database/{ddl.py => models.py} | 59 ++++++++++++------------------ 1 file changed, 23 insertions(+), 36 deletions(-) rename src/database/{ddl.py => models.py} (86%) diff --git a/src/database/ddl.py b/src/database/models.py similarity index 86% rename from src/database/ddl.py rename to src/database/models.py index 0983b74..0658c33 100644 --- a/src/database/ddl.py +++ b/src/database/models.py @@ -12,24 +12,15 @@ from src.server.validation import is_valid_base_comment from src.misc import clean -def get_database_connection(): - # for now it's an sqlite database - db = SqliteDatabase() - return db +def get_database_connection(dbms, db_name, **params): + if dbms == 'mysql': + return MySQLDatabase(db_name, **params) + else: + # return SqliteDatabase('/home/oleg/PycharmProjects/comment-server/database/default_pw.db') + return SqliteDatabase(db_name) - -database_proxy = DatabaseProxy() -database = get_database_connection() -database_proxy.initialize(database) - - -class BaseModel(Model): - class Meta: - database = database_proxy - - -class Channel(BaseModel): +class Channel(Model): claim_id = TextField(column_name='ClaimId', primary_key=True) name = TextField(column_name='Name') @@ -37,7 +28,7 @@ class Channel(BaseModel): table_name = 'CHANNEL' -class Comment(BaseModel): +class Comment(Model): comment = TextField(column_name='Body') channel = ForeignKeyField( backref='comments', @@ -47,7 +38,7 @@ class Comment(BaseModel): null=True ) comment_id = TextField(column_name='CommentId', primary_key=True) - is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT FALSE")]) + is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")]) claim_id = TextField(column_name='LbryClaimId') parent = ForeignKeyField( column_name='ParentId', @@ -63,7 +54,7 @@ class Comment(BaseModel): class Meta: table_name = 'COMMENT' indexes = ( - (('author', 'comment_id'), False), + (('channel', 'comment_id'), False), (('claim_id', 'comment_id'), False), ) @@ -203,8 +194,7 @@ def create_comment(comment: str = None, claim_id: str = None, timestamp = int(time.time()) comment_id = create_comment_id(comment, channel_id, timestamp) - with database_proxy.atomic(): - new_comment = Comment.create( + new_comment = Comment.create( claim_id=claim_id, comment_id=comment_id, comment=comment, @@ -214,7 +204,7 @@ def create_comment(comment: str = None, claim_id: str = None, signing_ts=signing_ts, timestamp=timestamp ) - return get_comment(new_comment.comment_id) + return get_comment(new_comment.comment_id) def delete_comment(comment_id: str) -> bool: @@ -223,8 +213,7 @@ def delete_comment(comment_id: str) -> bool: except DoesNotExist as e: raise ValueError from e else: - with database_proxy.atomic(): - return 0 < comment.delete_instance(True, delete_nullable=True) + return 0 < comment.delete_instance(True, delete_nullable=True) def edit_comment(comment_id: str, new_comment: str, new_sig: str, new_ts: str) -> bool: @@ -233,23 +222,21 @@ def edit_comment(comment_id: str, new_comment: str, new_sig: str, new_ts: str) - except DoesNotExist as e: raise ValueError from e else: - with database_proxy.atomic(): - comment.comment = new_comment - comment.signature = new_sig - comment.signing_ts = new_ts + comment.comment = new_comment + comment.signature = new_sig + comment.signing_ts = new_ts - # todo: add a 'last-modified' timestamp - comment.timestamp = int(time.time()) - return comment.save() > 0 + # todo: add a 'last-modified' timestamp + comment.timestamp = int(time.time()) + return comment.save() > 0 def set_hidden_flag(comment_ids: typing.List[str], hidden=True) -> bool: # sets `is_hidden` flag for all `comment_ids` to the `hidden` param - with database_proxy.atomic(): - update = (Comment - .update(is_hidden=hidden) - .where(Comment.comment_id.in_(comment_ids))) - return update.execute() > 0 + update = (Comment + .update(is_hidden=hidden) + .where(Comment.comment_id.in_(comment_ids))) + return update.execute() > 0 if __name__ == '__main__': From 45733d2dc4962714508ff45e3af3dc027f35ed49 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 30 Mar 2020 21:58:37 -0400 Subject: [PATCH 20/46] Remove queries.py --- src/database/queries.py | 292 ---------------------------------------- 1 file changed, 292 deletions(-) delete mode 100644 src/database/queries.py diff --git a/src/database/queries.py b/src/database/queries.py deleted file mode 100644 index 662c22f..0000000 --- a/src/database/queries.py +++ /dev/null @@ -1,292 +0,0 @@ -import atexit -import logging -import math -import sqlite3 -import time -import typing - -import nacl.hash - -from src.database.schema import CREATE_TABLES_QUERY - -logger = logging.getLogger(__name__) - -SELECT_COMMENTS_ON_CLAIMS = """ - SELECT comment, comment_id, claim_id, timestamp, is_hidden, parent_id, - channel_name, channel_id, channel_url, signature, signing_ts - FROM COMMENTS_ON_CLAIMS -""" - - -def clean(thing: dict) -> dict: - if 'is_hidden' in thing: - thing.update({'is_hidden': bool(thing['is_hidden'])}) - return {k: v for k, v in thing.items() if v is not None} - - -def obtain_connection(filepath: str = None, row_factory: bool = True): - connection = sqlite3.connect(filepath) - if row_factory: - connection.row_factory = sqlite3.Row - return connection - - -def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = None, - page: int = 1, page_size: int = 50, top_level=False): - with conn: - if top_level: - # doesn't include any - results = [clean(dict(row)) for row in conn.execute( - SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?", - (claim_id, page_size, page_size * (page - 1)) - )] - count = conn.execute( - "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id IS NULL", - (claim_id,) - ) - elif parent_id is None: - # include all, no specific parent comment - results = [clean(dict(row)) for row in conn.execute( - SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ", - (claim_id, page_size, page_size * (page - 1)) - )] - count = conn.execute( - "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ?", - (claim_id,) - ) - else: - results = [clean(dict(row)) for row in conn.execute( - SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? ", - (claim_id, parent_id, page_size, page_size * (page - 1)) - )] - count = conn.execute( - "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id = ?", - (claim_id, parent_id) - ) - count = tuple(count.fetchone())[0] - return { - 'items': results, - 'page': page, - 'page_size': page_size, - 'total_pages': math.ceil(count / page_size), - 'total_items': count, - 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id) - } - - -def get_claim_hidden_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50): - with conn: - results = conn.execute( - SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden IS ? LIMIT ? OFFSET ?", - (claim_id, hidden, page_size, page_size * (page - 1)) - ) - count = conn.execute( - "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden IS ?", (claim_id, hidden) - ) - results = [clean(dict(row)) for row in results.fetchall()] - count = tuple(count.fetchone())[0] - - return { - 'items': results, - 'page': page, - 'page_size': page_size, - 'total_pages': math.ceil(count / page_size), - 'total_items': count, - 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id) - } - - -def claim_has_hidden_comments(conn, claim_id): - with conn: - result = conn.execute( - "SELECT COUNT(DISTINCT is_hidden) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden IS 1", - (claim_id,) - ) - return bool(tuple(result.fetchone())[0]) - - -def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, - channel_id: str = None, signature: str = None, signing_ts: str = None, **extra) -> str: - timestamp = int(time.time()) - prehash = b':'.join((claim_id.encode(), comment.encode(), str(timestamp).encode(),)) - comment_id = nacl.hash.sha256(prehash).decode() - with conn: - curs = conn.execute( - """ - INSERT INTO COMMENT(CommentId, LbryClaimId, ChannelId, Body, ParentId, - Timestamp, Signature, SigningTs, IsHidden) - VALUES (:comment_id, :claim_id, :channel_id, :comment, NULL, - :timestamp, :signature, :signing_ts, 0) """, - { - 'comment_id': comment_id, - 'claim_id': claim_id, - 'channel_id': channel_id, - 'comment': comment, - 'timestamp': timestamp, - 'signature': signature, - 'signing_ts': signing_ts - } - ) - logging.info('attempted to insert comment with comment_id [%s] | %d rows affected', comment_id, curs.rowcount) - return comment_id - - -def insert_reply(conn: sqlite3.Connection, comment: str, parent_id: str, - channel_id: str = None, signature: str = None, - signing_ts: str = None, **extra) -> str: - timestamp = int(time.time()) - prehash = b':'.join((parent_id.encode(), comment.encode(), str(timestamp).encode(),)) - comment_id = nacl.hash.sha256(prehash).decode() - with conn: - curs = conn.execute( - """ - INSERT INTO COMMENT - (CommentId, LbryClaimId, ChannelId, Body, ParentId, Signature, Timestamp, SigningTs, IsHidden) - SELECT :comment_id, LbryClaimId, :channel_id, :comment, :parent_id, :signature, :timestamp, :signing_ts, 0 - FROM COMMENT WHERE CommentId = :parent_id - """, { - 'comment_id': comment_id, - 'parent_id': parent_id, - 'timestamp': timestamp, - 'comment': comment, - 'channel_id': channel_id, - 'signature': signature, - 'signing_ts': signing_ts - } - ) - logging.info('attempted to insert reply with comment_id [%s] | %d rows affected', comment_id, curs.rowcount) - return comment_id - - -def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict: - with conn: - curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,)) - thing = curry.fetchone() - return clean(dict(thing)) if thing else None - - -def get_comment_ids(conn: sqlite3.Connection, claim_id: str, parent_id: str = None, page=1, page_size=50): - """ Just return a list of the comment IDs that are associated with the given claim_id. - If get_all is specified then it returns all the IDs, otherwise only the IDs at that level. - if parent_id is left null then it only returns the top level comments. - - For pagination the parameters are: - get_all XOR (page_size + page) - """ - with conn: - if parent_id is None: - curs = conn.execute(""" - SELECT comment_id FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ? - """, (claim_id, page_size, page_size * abs(page - 1),) - ) - else: - curs = conn.execute(""" - SELECT comment_id FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? - """, (claim_id, parent_id, page_size, page_size * abs(page - 1),) - ) - return [tuple(row)[0] for row in curs.fetchall()] - - -def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.Union[list, None]: - """ Returns a list containing the comment data associated with each ID within the list""" - # format the input, under the assumption that the - placeholders = ', '.join('?' for _ in comment_ids) - with conn: - return [clean(dict(row)) for row in conn.execute( - SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})', - tuple(comment_ids) - )] - - -def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str) -> bool: - with conn: - curs = conn.execute("DELETE FROM COMMENT WHERE CommentId = ?", (comment_id,)) - return bool(curs.rowcount) - - -def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str): - with conn: - curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', (channel_id, channel_name)) - return bool(curs.rowcount) - - -def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str): - with conn: - channel = conn.execute( - "SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?", (comment_id,) - ).fetchone() - return dict(channel) if channel else {} - - -def get_claim_ids_from_comment_ids(conn: sqlite3.Connection, comment_ids: list): - with conn: - cids = conn.execute( - f""" SELECT CommentId as comment_id, LbryClaimId AS claim_id FROM COMMENT - WHERE CommentId IN ({', '.join('?' for _ in comment_ids)}) """, - tuple(comment_ids) - ) - return {row['comment_id']: row['claim_id'] for row in cids.fetchall()} - - -def hide_comments_by_id(conn: sqlite3.Connection, comment_ids: list) -> bool: - with conn: - curs = conn.cursor() - curs.executemany( - "UPDATE COMMENT SET IsHidden = 1 WHERE CommentId = ?", - [[c] for c in comment_ids] - ) - return bool(curs.rowcount) - - -def edit_comment_by_id(conn: sqlite3.Connection, comment_id: str, comment: str, - signature: str, signing_ts: str) -> bool: - with conn: - curs = conn.execute( - """ - UPDATE COMMENT - SET Body = :comment, Signature = :signature, SigningTs = :signing_ts - WHERE CommentId = :comment_id - """, - { - 'comment': comment, - 'signature': signature, - 'signing_ts': signing_ts, - 'comment_id': comment_id - }) - logger.info("updated comment with `comment_id`: %s", comment_id) - return bool(curs.rowcount) - - -class DatabaseWriter(object): - _writer = None - - def __init__(self, db_file): - if not DatabaseWriter._writer: - self.conn = obtain_connection(db_file) - DatabaseWriter._writer = self - atexit.register(self.cleanup) - logging.info('Database writer has been created at %s', repr(self)) - else: - logging.warning('Someone attempted to insantiate DatabaseWriter') - raise TypeError('Database Writer already exists!') - - def cleanup(self): - logging.info('Cleaning up database writer') - self.conn.close() - DatabaseWriter._writer = None - - @property - def connection(self): - return self.conn - - -def setup_database(db_path): - with sqlite3.connect(db_path) as conn: - conn.executescript(CREATE_TABLES_QUERY) - - -def backup_database(conn: sqlite3.Connection, back_fp): - with sqlite3.connect(back_fp) as back: - conn.backup(back) From dba14460cc8845e707a6e552d36d1bf4da9c118d Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 30 Mar 2020 21:59:26 -0400 Subject: [PATCH 21/46] Unittests using peewee binds instead of direct sqlite connection --- test/test_database.py | 222 +++++++++++++++++++++++++----------------- test/testcase.py | 49 ++++++---- 2 files changed, 160 insertions(+), 111 deletions(-) diff --git a/test/test_database.py b/test/test_database.py index c43bb0a..bf25bba 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -1,18 +1,13 @@ -import sqlite3 - from random import randint import faker from faker.providers import internet from faker.providers import lorem from faker.providers import misc -from src.database.queries import get_comments_by_id -from src.database.queries import get_comment_ids -from src.database.queries import get_claim_comments -from src.database.queries import get_claim_hidden_comments -from src.database.writes import create_comment_or_error -from src.database.queries import hide_comments_by_id -from src.database.queries import delete_comment_by_id +from src.database.models import create_comment +from src.database.models import delete_comment +from src.database.models import comment_list, get_comment, get_comments_by_id +from src.database.models import set_hidden_flag from test.testcase import DatabaseTestCase fake = faker.Faker() @@ -27,26 +22,25 @@ class TestDatabaseOperations(DatabaseTestCase): self.claimId = '529357c3422c6046d3fec76be2358004ba22e340' def test01NamedComments(self): - comment = create_comment_or_error( - conn=self.conn, + comment = create_comment( claim_id=self.claimId, comment='This is a named comment', channel_name='@username', channel_id='529357c3422c6046d3fec76be2358004ba22abcd', - signature=fake.uuid4(), + signature='22'*64, signing_ts='aaa' ) self.assertIsNotNone(comment) self.assertNotIn('parent_in', comment) + previous_id = comment['comment_id'] - reply = create_comment_or_error( - conn=self.conn, + reply = create_comment( claim_id=self.claimId, comment='This is a named response', channel_name='@another_username', channel_id='529357c3422c6046d3fec76be2358004ba224bcd', parent_id=previous_id, - signature=fake.uuid4(), + signature='11'*64, signing_ts='aaa' ) self.assertIsNotNone(reply) @@ -54,34 +48,32 @@ class TestDatabaseOperations(DatabaseTestCase): def test02AnonymousComments(self): self.assertRaises( - sqlite3.IntegrityError, - create_comment_or_error, - conn=self.conn, + ValueError, + create_comment, claim_id=self.claimId, comment='This is an ANONYMOUS comment' ) def test03SignedComments(self): - comment = create_comment_or_error( - conn=self.conn, + comment = create_comment( claim_id=self.claimId, comment='I like big butts and i cannot lie', channel_name='@sirmixalot', channel_id='529357c3422c6046d3fec76be2358005ba22abcd', - signature=fake.uuid4(), + signature='24'*64, signing_ts='asdasd' ) self.assertIsNotNone(comment) self.assertIn('signing_ts', comment) + previous_id = comment['comment_id'] - reply = create_comment_or_error( - conn=self.conn, + reply = create_comment( claim_id=self.claimId, comment='This is a LBRY verified response', channel_name='@LBRY', channel_id='529357c3422c6046d3fec76be2358001ba224bcd', parent_id=previous_id, - signature=fake.uuid4(), + signature='12'*64, signing_ts='sfdfdfds' ) self.assertIsNotNone(reply) @@ -90,75 +82,109 @@ class TestDatabaseOperations(DatabaseTestCase): def test04UsernameVariations(self): self.assertRaises( - AssertionError, - callable=create_comment_or_error, - conn=self.conn, + ValueError, + create_comment, claim_id=self.claimId, channel_name='$#(@#$@#$', channel_id='529357c3422c6046d3fec76be2358001ba224b23', - comment='this is an invalid username' + comment='this is an invalid username', + signature='1' * 128, + signing_ts='123' ) - valid_username = create_comment_or_error( - conn=self.conn, + + valid_username = create_comment( claim_id=self.claimId, channel_name='@' + 'a' * 255, channel_id='529357c3422c6046d3fec76be2358001ba224b23', - comment='this is a valid username' + comment='this is a valid username', + signature='1'*128, + signing_ts='123' ) self.assertIsNotNone(valid_username) - self.assertRaises(AssertionError, - callable=create_comment_or_error, - conn=self.conn, - claim_id=self.claimId, - channel_name='@' + 'a' * 256, - channel_id='529357c3422c6046d3fec76be2358001ba224b23', - comment='this username is too long' - ) self.assertRaises( - AssertionError, - callable=create_comment_or_error, - conn=self.conn, + ValueError, + create_comment, + claim_id=self.claimId, + channel_name='@' + 'a' * 256, + channel_id='529357c3422c6046d3fec76be2358001ba224b23', + comment='this username is too long', + signature='2' * 128, + signing_ts='123' + ) + + self.assertRaises( + ValueError, + create_comment, claim_id=self.claimId, channel_name='', channel_id='529357c3422c6046d3fec76be2358001ba224b23', - comment='this username should not default to ANONYMOUS' + comment='this username should not default to ANONYMOUS', + signature='3' * 128, + signing_ts='123' ) + self.assertRaises( - AssertionError, - callable=create_comment_or_error, - conn=self.conn, + ValueError, + create_comment, claim_id=self.claimId, channel_name='@', channel_id='529357c3422c6046d3fec76be2358001ba224b23', - comment='this username is too short' + comment='this username is too short', + signature='3' * 128, + signing_ts='123' ) def test05HideComments(self): - comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') - comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + comm = create_comment( + comment='Comment #1', + claim_id=self.claimId, + channel_id='1'*40, + channel_name='@Doge123', + signature='a'*128, + signing_ts='123' + ) + comment = get_comment(comm['comment_id']) self.assertFalse(comment['is_hidden']) - success = hide_comments_by_id(self.conn, [comm['comment_id']]) + + success = set_hidden_flag([comm['comment_id']]) self.assertTrue(success) - comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + + comment = get_comment(comm['comment_id']) self.assertTrue(comment['is_hidden']) - success = hide_comments_by_id(self.conn, [comm['comment_id']]) + + success = set_hidden_flag([comm['comment_id']]) self.assertTrue(success) - comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + + comment = get_comment(comm['comment_id']) self.assertTrue(comment['is_hidden']) def test06DeleteComments(self): - comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') - comments = get_claim_comments(self.conn, self.claimId) - match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items'])) - self.assertTrue(match) - deleted = delete_comment_by_id(self.conn, comm['comment_id']) + # make sure that the comment was created + comm = create_comment( + comment='Comment #1', + claim_id=self.claimId, + channel_id='1'*40, + channel_name='@Doge123', + signature='a'*128, + signing_ts='123' + ) + comments = comment_list(self.claimId) + match = [x for x in comments['items'] if x['comment_id'] == comm['comment_id']] + self.assertTrue(len(match) > 0) + + deleted = delete_comment(comm['comment_id']) self.assertTrue(deleted) - comments = get_claim_comments(self.conn, self.claimId) - match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items'])) + + # make sure that we can't find the comment here + comments = comment_list(self.claimId) + match = [x for x in comments['items'] if x['comment_id'] == comm['comment_id']] self.assertFalse(match) - deleted = delete_comment_by_id(self.conn, comm['comment_id']) - self.assertFalse(deleted) + self.assertRaises( + ValueError, + delete_comment, + comment_id=comm['comment_id'], + ) class ListDatabaseTest(DatabaseTestCase): @@ -169,61 +195,75 @@ class ListDatabaseTest(DatabaseTestCase): def testLists(self): for claim_id in self.claim_ids: with self.subTest(claim_id=claim_id): - comments = get_claim_comments(self.conn, claim_id) + comments = comment_list(claim_id) self.assertIsNotNone(comments) self.assertGreater(comments['page_size'], 0) self.assertIn('has_hidden_comments', comments) self.assertFalse(comments['has_hidden_comments']) - top_comments = get_claim_comments(self.conn, claim_id, top_level=True, page=1, page_size=50) + top_comments = comment_list(claim_id, top_level=True, page=1, page_size=50) self.assertIsNotNone(top_comments) self.assertEqual(top_comments['page_size'], 50) self.assertEqual(top_comments['page'], 1) self.assertGreaterEqual(top_comments['total_pages'], 0) self.assertGreaterEqual(top_comments['total_items'], 0) - comment_ids = get_comment_ids(self.conn, claim_id, page_size=50, page=1) + comment_ids = comment_list(claim_id, page_size=50, page=1) with self.subTest(comment_ids=comment_ids): self.assertIsNotNone(comment_ids) self.assertLessEqual(len(comment_ids), 50) - matching_comments = get_comments_by_id(self.conn, comment_ids) + matching_comments = (comment_ids) self.assertIsNotNone(matching_comments) self.assertEqual(len(matching_comments), len(comment_ids)) def testHiddenCommentLists(self): claim_id = 'a'*40 - comm1 = create_comment_or_error(self.conn, 'Comment #1', claim_id, '1'*40, '@Doge123', 'a'*128, '123') - comm2 = create_comment_or_error(self.conn, 'Comment #2', claim_id, '1'*40, '@Doge123', 'b'*128, '123') - comm3 = create_comment_or_error(self.conn, 'Comment #3', claim_id, '1'*40, '@Doge123', 'c'*128, '123') + comm1 = create_comment( + 'Comment #1', + claim_id, + channel_id='1'*40, + channel_name='@Doge123', + signature='a'*128, + signing_ts='123' + ) + comm2 = create_comment( + 'Comment #2', claim_id, + channel_id='1'*40, + channel_name='@Doge123', + signature='b'*128, + signing_ts='123' + ) + comm3 = create_comment( + 'Comment #3', claim_id, + channel_id='1'*40, + channel_name='@Doge123', + signature='c'*128, + signing_ts='123' + ) comments = [comm1, comm2, comm3] - comment_list = get_claim_comments(self.conn, claim_id) - self.assertIn('items', comment_list) - self.assertIn('has_hidden_comments', comment_list) - self.assertEqual(len(comments), comment_list['total_items']) - self.assertIn('has_hidden_comments', comment_list) - self.assertFalse(comment_list['has_hidden_comments']) - hide_comments_by_id(self.conn, [comm2['comment_id']]) + listed_comments = comment_list(claim_id) + self.assertEqual(len(comments), listed_comments['total_items']) + self.assertFalse(listed_comments['has_hidden_comments']) - default_comments = get_claim_hidden_comments(self.conn, claim_id) - self.assertIn('has_hidden_comments', default_comments) + set_hidden_flag([comm2['comment_id']]) + hidden = comment_list(claim_id, exclude_mode='hidden') - hidden_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=True) - self.assertIn('has_hidden_comments', hidden_comments) - self.assertEqual(default_comments, hidden_comments) + self.assertTrue(hidden['has_hidden_comments']) + self.assertGreater(len(hidden['items']), 0) - hidden_comment = hidden_comments['items'][0] + visible = comment_list(claim_id, exclude_mode='visible') + self.assertFalse(visible['has_hidden_comments']) + self.assertNotEqual(listed_comments['items'], visible['items']) + + # make sure the hidden comment is the one we marked as hidden + hidden_comment = hidden['items'][0] self.assertEqual(hidden_comment['comment_id'], comm2['comment_id']) - visible_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=False) - self.assertIn('has_hidden_comments', visible_comments) - self.assertNotIn(hidden_comment, visible_comments['items']) - - hidden_ids = [c['comment_id'] for c in hidden_comments['items']] - visible_ids = [c['comment_id'] for c in visible_comments['items']] + hidden_ids = [c['comment_id'] for c in hidden['items']] + visible_ids = [c['comment_id'] for c in visible['items']] composite_ids = hidden_ids + visible_ids + listed_comments = comment_list(claim_id) + all_ids = [c['comment_id'] for c in listed_comments['items']] composite_ids.sort() - - comment_list = get_claim_comments(self.conn, claim_id) - all_ids = [c['comment_id'] for c in comment_list['items']] all_ids.sort() self.assertEqual(composite_ids, all_ids) diff --git a/test/testcase.py b/test/testcase.py index ddd8b44..12e1197 100644 --- a/test/testcase.py +++ b/test/testcase.py @@ -1,12 +1,39 @@ import os import pathlib import unittest -from asyncio.runners import _cancel_all_tasks # type: ignore from unittest.case import _Outcome import asyncio +from asyncio.runners import _cancel_all_tasks # type: ignore +from peewee import * + +from src.database.models import Channel, Comment + + +test_db = SqliteDatabase(':memory:') + + +MODELS = [Channel, Comment] + + +class DatabaseTestCase(unittest.TestCase): + def __init__(self, methodName='DatabaseTest'): + super().__init__(methodName) + + def setUp(self) -> None: + super().setUp() + test_db.bind(MODELS, bind_refs=False, bind_backrefs=False) + + test_db.connect() + test_db.create_tables(MODELS) + + def tearDown(self) -> None: + # drop tables for next test + test_db.drop_tables(MODELS) + + # close connection + test_db.close() -from src.database.queries import obtain_connection, setup_database class AsyncioTestCase(unittest.TestCase): @@ -117,21 +144,3 @@ class AsyncioTestCase(unittest.TestCase): self.loop.run_until_complete(maybe_coroutine) -class DatabaseTestCase(unittest.TestCase): - db_file = 'test.db' - - def __init__(self, methodName='DatabaseTest'): - super().__init__(methodName) - if pathlib.Path(self.db_file).exists(): - os.remove(self.db_file) - - def setUp(self) -> None: - super().setUp() - setup_database(self.db_file) - self.conn = obtain_connection(self.db_file) - self.addCleanup(self.conn.close) - self.addCleanup(os.remove, self.db_file) - - def tearDown(self) -> None: - self.conn.close() - From e0b6d16c89eed19f3053bb75a2ac675ba69ed48e Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 30 Mar 2020 22:00:10 -0400 Subject: [PATCH 22/46] Removes certain sqlite-specific functions from application --- src/server/app.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/server/app.py b/src/server/app.py index 446c378..35471fd 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -10,7 +10,7 @@ import aiojobs.aiohttp from aiohttp import web from src.database.queries import obtain_connection, DatabaseWriter -from src.database.queries import setup_database, backup_database +from src.database.queries import setup_database from src.server.handles import api_endpoint, get_api_endpoint logger = logging.getLogger(__name__) @@ -24,21 +24,9 @@ async def setup_db_schema(app): logger.info(f'Database already exists in {app["db_path"]}, skipping setup') -async def database_backup_routine(app): - try: - while True: - await asyncio.sleep(app['config']['backup_int']) - with app['reader'] as conn: - logger.debug('backing up database') - backup_database(conn, app['backup']) - except asyncio.CancelledError: - pass - - async def start_background_tasks(app): # Reading the DB app['reader'] = obtain_connection(app['db_path'], True) - app['waitful_backup'] = asyncio.create_task(database_backup_routine(app)) # Scheduler to prevent multiple threads from writing to DB simulataneously app['comment_scheduler'] = await aiojobs.create_scheduler(limit=1, pending_limit=0) @@ -50,9 +38,6 @@ async def start_background_tasks(app): async def close_database_connections(app): - logger.info('Ending background backup loop') - app['waitful_backup'].cancel() - await app['waitful_backup'] app['reader'].close() app['writer'].close() app['db_writer'].cleanup() @@ -67,7 +52,7 @@ async def close_schedulers(app): class CommentDaemon: - def __init__(self, config, db_file=None, backup=None, **kwargs): + def __init__(self, config, db_file=None, **kwargs): app = web.Application() # configure the config @@ -76,8 +61,6 @@ class CommentDaemon: # configure the db file app['db_path'] = db_file or config.get('db_path') - if app['db_path']: - app['backup'] = backup or '.'.join((app['db_path'], 'backup')) # configure the order of tasks to run during app lifetime app.on_startup.append(setup_db_schema) From 115371163609db3ea7689c92e31432888388cd34 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 31 Mar 2020 13:55:44 -0400 Subject: [PATCH 23/46] Reimplements API methods using ORM --- src/database/models.py | 31 --------- src/server/handles.py | 154 +++++++++++++++++++++++++++++++++-------- 2 files changed, 126 insertions(+), 59 deletions(-) diff --git a/src/database/models.py b/src/database/models.py index 0658c33..43a1bbb 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -126,37 +126,6 @@ def get_comment(comment_id: str) -> dict: .pop()) -def get_comment_ids(claim_id: str = None, parent_id: str = None, - page: int = 1, page_size: int = 50, flattened=False) -> dict: - results = comment_list( - claim_id, parent_id, - top_level=(parent_id is None), - page=page, page_size=page_size, - select_fields=['comment_id', 'parent_id'] - ) - if flattened: - results.update({ - 'items': [item['comment_id'] for item in results['items']], - 'replies': [(item['comment_id'], item.get('parent_id')) for item in results['items']] - }) - return results - - -def get_comments_by_id(comment_ids: typing.Union[list, tuple]) -> dict: - expression = Comment.comment_id.in_(comment_ids) - return comment_list(expressions=expression, page_size=len(comment_ids)) - - -def get_channel_from_comment_id(comment_id: str) -> dict: - results = comment_list( - expressions=(Comment.comment_id == comment_id), - select_fields=['channel_name', 'channel_id', 'channel_url'], - page_size=1 - ) - # todo: make the return type here consistent - return results['items'].pop() - - def create_comment_id(comment: str, channel_id: str, timestamp: int): # We convert the timestamp from seconds into minutes # to prevent spammers from commenting the same BS everywhere. diff --git a/src/server/handles.py b/src/server/handles.py index a63504b..bcc2fa6 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -1,16 +1,22 @@ import asyncio import logging import time +import typing from aiohttp import web from aiojobs.aiohttp import atomic -import src.database.queries as db -from src.database.writes import abandon_comment, create_comment -from src.database.writes import hide_comments -from src.database.writes import edit_comment -from src.misc import clean_input_params +from src.server.validation import validate_signature_from_claim +from src.misc import clean_input_params, get_claim_from_id from src.server.errors import make_error, report_error +from src.database.models import Comment, Channel +from src.database.models import get_comment +from src.database.models import comment_list +from src.database.models import create_comment +from src.database.models import edit_comment +from src.database.models import delete_comment +from src.database.models import set_hidden_flag + logger = logging.getLogger(__name__) @@ -20,37 +26,127 @@ def ping(*args): return 'pong' -def handle_get_channel_from_comment_id(app, kwargs: dict): - return db.get_channel_id_from_comment_id(app['reader'], **kwargs) +def handle_get_channel_from_comment_id(app: web.Application, comment_id: str) -> dict: + comment = get_comment(comment_id) + return { + 'channel_id': comment['channel_id'], + 'channel_name': comment['channel_name'] + } -def handle_get_comment_ids(app, kwargs): - return db.get_comment_ids(app['reader'], **kwargs) +def handle_get_comment_ids( + app: web.Application, + claim_id: str, + parent_id: str = None, + page: int = 1, + page_size: int = 50, + flattened=False +) -> dict: + results = comment_list( + claim_id=claim_id, + parent_id=parent_id, + top_level=(parent_id is None), + page=page, + page_size=page_size, + select_fields=['comment_id', 'parent_id'] + ) + if flattened: + results.update({ + 'items': [item['comment_id'] for item in results['items']], + 'replies': [(item['comment_id'], item.get('parent_id')) + for item in results['items']] + }) + return results -def handle_get_claim_comments(app, kwargs): - return db.get_claim_comments(app['reader'], **kwargs) +def handle_get_comments_by_id( + app: web.Application, + comment_ids: typing.Union[list, tuple] +) -> dict: + expression = Comment.comment_id.in_(comment_ids) + return comment_list(expressions=expression, page_size=len(comment_ids)) -def handle_get_comments_by_id(app, kwargs): - return db.get_comments_by_id(app['reader'], **kwargs) +def handle_get_claim_comments( + app: web.Application, + claim_id: str, + parent_id: str = None, + page: int = 1, + page_size: int = 50, + top_level: bool = False +) -> dict: + return comment_list( + claim_id=claim_id, + parent_id=parent_id, + page=page, + page_size=page_size, + top_level=top_level + ) -def handle_get_claim_hidden_comments(app, kwargs): - return db.get_claim_hidden_comments(app['reader'], **kwargs) +def handle_get_claim_hidden_comments( + app: web.Application, + claim_id: str, + hidden: bool, + page: int = 1, + page_size: int = 50, +) -> dict: + exclude = 'hidden' if hidden else 'visible' + return comment_list( + claim_id=claim_id, + exclude_mode=exclude, + page=page, + page_size=page_size + ) + + +def get_channel_from_comment_id(app, comment_id: str) -> dict: + results = comment_list( + expressions=(Comment.comment_id == comment_id), + select_fields=['channel_name', 'channel_id', 'channel_url'], + page_size=1 + ) + # todo: make the return type here consistent + return results['items'].pop() async def handle_abandon_comment(app, params): - return {'abandoned': await abandon_comment(app, **params)} + # return {'abandoned': await abandon_comment(app, **params)} + raise NotImplementedError -async def handle_hide_comments(app, params): - return {'hidden': await hide_comments(app, **params)} +async def handle_hide_comments(app, pieces: list = None, claim_id: str = None) -> dict: + + # return {'hidden': await hide_comments(app, **params)} + raise NotImplementedError -async def handle_edit_comment(app, params): - if await edit_comment(app, **params): - return db.get_comment_or_none(app['reader'], params['comment_id']) +async def handle_edit_comment(app, comment: str = None, comment_id: str = None, + signature: str = None, signing_ts: str = None, **params) -> dict: + current = get_comment(comment_id) + channel_claim = await get_claim_from_id(app, current['channel_id']) + if not validate_signature_from_claim(channel_claim, signature, signing_ts, comment): + raise ValueError('Signature could not be validated') + + with app['db'].atomic(): + if not edit_comment(comment_id, comment, signature, signing_ts): + raise ValueError('Comment could not be edited') + return get_comment(comment_id) + + +def handle_create_comment(app, comment: str = None, claim_id: str = None, + parent_id: str = None, channel_id: str = None, channel_name: str = None, + signature: str = None, signing_ts: str = None) -> dict: + with app['db'].atomic(): + return create_comment( + comment=comment, + claim_id=claim_id, + parent_id=parent_id, + channel_id=channel_id, + channel_name=channel_name, + signature=signature, + signing_ts=signing_ts + ) METHODS = { @@ -59,8 +155,8 @@ METHODS = { 'get_claim_hidden_comments': handle_get_claim_hidden_comments, # this gets used 'get_comment_ids': handle_get_comment_ids, 'get_comments_by_id': handle_get_comments_by_id, # this gets used - 'get_channel_from_comment_id': handle_get_channel_from_comment_id, # this gets used - 'create_comment': create_comment, # this gets used + 'get_channel_from_comment_id': get_channel_from_comment_id, # this gets used + 'create_comment': handle_create_comment, # this gets used 'delete_comment': handle_abandon_comment, 'abandon_comment': handle_abandon_comment, # this gets used 'hide_comments': handle_hide_comments, # this gets used @@ -78,17 +174,19 @@ async def process_json(app, body: dict) -> dict: start = time.time() try: if asyncio.iscoroutinefunction(METHODS[method]): - result = await METHODS[method](app, params) + result = await METHODS[method](app, **params) else: - result = METHODS[method](app, params) - response['result'] = result + result = METHODS[method](app, **params) + except Exception as err: - logger.exception(f'Got {type(err).__name__}:') + logger.exception(f'Got {type(err).__name__}:\n{err}') if type(err) in (ValueError, TypeError): # param error, not too important response['error'] = make_error('INVALID_PARAMS', err) else: response['error'] = make_error('INTERNAL', err) - await app['webhooks'].spawn(report_error(app, err, body)) + await app['webhooks'].spawn(report_error(app, err, body)) + else: + response['result'] = result finally: end = time.time() From bd06d1c992e25361b576489a5638007a9b929753 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 31 Mar 2020 13:57:30 -0400 Subject: [PATCH 24/46] Improves database configuration --- setup.py | 7 ++-- src/database/models.py | 13 +------- src/main.py | 12 +++++-- src/server/app.py | 72 ++++++++++++++++++++++-------------------- test/test_database.py | 2 +- test/test_server.py | 26 +++++++-------- test/testcase.py | 1 - 7 files changed, 65 insertions(+), 68 deletions(-) diff --git a/setup.py b/setup.py index 25087e4..c8755cf 100644 --- a/setup.py +++ b/setup.py @@ -17,12 +17,11 @@ setup( 'mysql-connector-python', 'pyyaml', 'Faker>=1.0.7', - 'asyncio>=3.4.3', - 'aiohttp==3.5.4', - 'aiojobs==0.2.2', + 'asyncio', + 'aiohttp', + 'aiojobs', 'ecdsa>=0.13.3', 'cryptography==2.5', - 'aiosqlite==0.10.0', 'PyNaCl>=1.3.0', 'requests', 'cython', diff --git a/src/database/models.py b/src/database/models.py index 43a1bbb..f999f34 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -12,14 +12,6 @@ from src.server.validation import is_valid_base_comment from src.misc import clean -def get_database_connection(dbms, db_name, **params): - if dbms == 'mysql': - return MySQLDatabase(db_name, **params) - else: - # return SqliteDatabase('/home/oleg/PycharmProjects/comment-server/database/default_pw.db') - return SqliteDatabase(db_name) - - class Channel(Model): claim_id = TextField(column_name='ClaimId', primary_key=True) name = TextField(column_name='Name') @@ -157,7 +149,7 @@ def create_comment(comment: str = None, claim_id: str = None, raise ValueError('Invalid Parameters given for comment') channel, _ = Channel.get_or_create(name=channel_name, claim_id=channel_id) - if parent_id: + if parent_id and not claim_id: parent: Comment = Comment.get_by_id(parent_id) claim_id = parent.claim_id @@ -219,7 +211,4 @@ if __name__ == '__main__': (Comment.claim_id ** '420%')) ) - ids = get_comment_ids('4207d2378bf4340e68c9d88faf7ee24ea1a1f95a') - print(json.dumps(comments, indent=4)) - print(json.dumps(ids, indent=4)) diff --git a/src/main.py b/src/main.py index c22a82b..124c7d6 100644 --- a/src/main.py +++ b/src/main.py @@ -84,12 +84,13 @@ def get_config(filepath): def setup_db_from_config(config: dict): - if 'sqlite' in config['database']: + mode = config['mode'] + if config[mode]['database'] == 'sqlite': if not os.path.exists(DATABASE_DIR): os.mkdir(DATABASE_DIR) - config['db_path'] = os.path.join( - DATABASE_DIR, config['database']['sqlite'] + config[mode]['db_file'] = os.path.join( + DATABASE_DIR, config[mode]['name'] ) @@ -98,10 +99,15 @@ def main(argv=None): parser = argparse.ArgumentParser(description='LBRY Comment Server') parser.add_argument('--port', type=int) parser.add_argument('--config', type=str) + parser.add_argument('--mode', type=str) args = parser.parse_args(argv) config = get_config(CONFIG_FILE) if not args.config else args.config setup_logging_from_config(config) + + if args.mode: + config['mode'] = args.mode + setup_db_from_config(config) if args.port: diff --git a/src/server/app.py b/src/server/app.py index 35471fd..389083e 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -1,7 +1,6 @@ # cython: language_level=3 import asyncio import logging -import pathlib import signal import time @@ -9,61 +8,67 @@ import aiojobs import aiojobs.aiohttp from aiohttp import web -from src.database.queries import obtain_connection, DatabaseWriter -from src.database.queries import setup_database +from peewee import * from src.server.handles import api_endpoint, get_api_endpoint +from src.database.models import Comment, Channel +MODELS = [Comment, Channel] logger = logging.getLogger(__name__) -async def setup_db_schema(app): - if not pathlib.Path(app['db_path']).exists(): - logger.info(f'Setting up schema in {app["db_path"]}') - setup_database(app['db_path']) - else: - logger.info(f'Database already exists in {app["db_path"]}, skipping setup') +def setup_database(app): + config = app['config'] + mode = config['mode'] + + # switch between Database objects + if config[mode]['database'] == 'mysql': + app['db'] = MySQLDatabase( + database=config[mode]['name'], + user=config[mode]['user'], + host=config[mode]['host'], + password=config[mode]['password'], + port=config[mode]['port'], + ) + elif config[mode]['database'] == 'sqlite': + app['db'] = SqliteDatabase( + config[mode]['file'], + pragmas=config[mode]['pragmas'] + ) + + # bind the Model list to the database + app['db'].bind(MODELS, bind_refs=False, bind_backrefs=False) async def start_background_tasks(app): - # Reading the DB - app['reader'] = obtain_connection(app['db_path'], True) - - # Scheduler to prevent multiple threads from writing to DB simulataneously - app['comment_scheduler'] = await aiojobs.create_scheduler(limit=1, pending_limit=0) - app['db_writer'] = DatabaseWriter(app['db_path']) - app['writer'] = app['db_writer'].connection + app['db'].connect() + app['db'].create_tables(MODELS) # for requesting to external and internal APIs app['webhooks'] = await aiojobs.create_scheduler(pending_limit=0) async def close_database_connections(app): - app['reader'].close() - app['writer'].close() - app['db_writer'].cleanup() + app['db'].close() async def close_schedulers(app): - logger.info('Closing comment_scheduler') - await app['comment_scheduler'].close() - logger.info('Closing scheduler for webhook requests') await app['webhooks'].close() class CommentDaemon: - def __init__(self, config, db_file=None, **kwargs): + def __init__(self, config, **kwargs): app = web.Application() + app['config'] = config # configure the config - app['config'] = config - self.config = app['config'] + self.config = config + self.host = config['host'] + self.port = config['port'] - # configure the db file - app['db_path'] = db_file or config.get('db_path') + setup_database(app) # configure the order of tasks to run during app lifetime - app.on_startup.append(setup_db_schema) app.on_startup.append(start_background_tasks) app.on_shutdown.append(close_schedulers) app.on_cleanup.append(close_database_connections) @@ -85,20 +90,19 @@ class CommentDaemon: await self.app_runner.setup() self.app_site = web.TCPSite( runner=self.app_runner, - host=host or self.config['host'], - port=port or self.config['port'], + host=host or self.host, + port=port or self.port, ) await self.app_site.start() - logger.info(f'Comment Server is running on {self.config["host"]}:{self.config["port"]}') + logger.info(f'Comment Server is running on {self.host}:{self.port}') async def stop(self): await self.app_runner.shutdown() await self.app_runner.cleanup() -def run_app(config, db_file=None): - comment_app = CommentDaemon(config=config, db_file=db_file, close_timeout=5.0) - +def run_app(config): + comment_app = CommentDaemon(config=config) loop = asyncio.get_event_loop() def __exit(): diff --git a/test/test_database.py b/test/test_database.py index bf25bba..c698ad1 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -6,7 +6,7 @@ from faker.providers import misc from src.database.models import create_comment from src.database.models import delete_comment -from src.database.models import comment_list, get_comment, get_comments_by_id +from src.database.models import comment_list, get_comment from src.database.models import set_hidden_flag from test.testcase import DatabaseTestCase diff --git a/test/test_server.py b/test/test_server.py index 55fcdba..56bccfb 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -17,6 +17,8 @@ from test.testcase import AsyncioTestCase config = get_config(CONFIG_FILE) +config['mode'] = 'testing' +config['testing']['file'] = ':memory:' if 'slack_webhook' in config: @@ -74,10 +76,10 @@ def create_test_comments(values: iter, **default): class ServerTest(AsyncioTestCase): - db_file = 'test.db' - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + config['mode'] = 'testing' + config['testing']['file'] = ':memory:' self.host = 'localhost' self.port = 5931 @@ -88,11 +90,10 @@ class ServerTest(AsyncioTestCase): @classmethod def tearDownClass(cls) -> None: print('exit reached') - os.remove(cls.db_file) async def asyncSetUp(self): await super().asyncSetUp() - self.server = app.CommentDaemon(config, db_file=self.db_file) + self.server = app.CommentDaemon(config) await self.server.start(host=self.host, port=self.port) self.addCleanup(self.server.stop) @@ -138,14 +139,16 @@ class ServerTest(AsyncioTestCase): test_all = create_test_comments(replace.keys(), **{ k: None for k in replace.keys() }) + test_all.reverse() for test in test_all: - with self.subTest(test=test): + nulls = 'null fields: ' + ', '.join(k for k, v in test.items() if not v) + with self.subTest(test=nulls): message = await self.post_comment(**test) self.assertTrue('result' in message or 'error' in message) if 'error' in message: - self.assertFalse(is_valid_base_comment(**test)) + self.assertFalse(is_valid_base_comment(**test, strict=True)) else: - self.assertTrue(is_valid_base_comment(**test)) + self.assertTrue(is_valid_base_comment(**test, strict=True)) async def test04CreateAllReplies(self): claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8' @@ -223,7 +226,8 @@ class ListCommentsTest(AsyncioTestCase): super().__init__(*args, **kwargs) self.host = 'localhost' self.port = 5931 - self.db_file = 'list_test.db' + config['mode'] = 'testing' + config['testing']['file'] = ':memory:' self.claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8' self.comment_ids = None @@ -234,10 +238,6 @@ class ListCommentsTest(AsyncioTestCase): async def post_comment(self, **params): return await jsonrpc_post(self.url, 'create_comment', **params) - def tearDown(self) -> None: - print('exit reached') - os.remove(self.db_file) - async def create_lots_of_comments(self, n=23): self.comment_list = [{key: self.replace[key]() for key in self.replace.keys()} for _ in range(23)] for comment in self.comment_list: @@ -247,7 +247,7 @@ class ListCommentsTest(AsyncioTestCase): async def asyncSetUp(self): await super().asyncSetUp() - self.server = app.CommentDaemon(config, db_file=self.db_file) + self.server = app.CommentDaemon(config) await self.server.start(self.host, self.port) self.addCleanup(self.server.stop) diff --git a/test/testcase.py b/test/testcase.py index 12e1197..415041d 100644 --- a/test/testcase.py +++ b/test/testcase.py @@ -35,7 +35,6 @@ class DatabaseTestCase(unittest.TestCase): test_db.close() - class AsyncioTestCase(unittest.TestCase): # Implementation inspired by discussion: # https://bugs.python.org/issue32972 From a6f056821f719d6c52a502c1f0ea050cb2c56f7d Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 31 Mar 2020 13:59:20 -0400 Subject: [PATCH 25/46] Adds a `strict` validator which fails upon possible FK violations --- src/server/validation.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/server/validation.py b/src/server/validation.py index ef1c841..3b251ba 100644 --- a/src/server/validation.py +++ b/src/server/validation.py @@ -51,11 +51,31 @@ def claim_id_is_valid(claim_id: str) -> bool: # default to None so params can be treated as kwargs; param count becomes more manageable -def is_valid_base_comment(comment: str = None, claim_id: str = None, parent_id: str = None, **kwargs) -> bool: - return comment and body_is_valid(comment) and \ - ((claim_id and claim_id_is_valid(claim_id)) or # parentid is used in place of claimid in replies - (parent_id and comment_id_is_valid(parent_id))) \ - and is_valid_credential_input(**kwargs) +def is_valid_base_comment( + comment: str = None, + claim_id: str = None, + parent_id: str = None, + strict: bool = False, + **kwargs, +) -> bool: + try: + assert comment and body_is_valid(comment) + # strict mode assumes that the parent_id might not exist + if strict: + assert claim_id and claim_id_is_valid(claim_id) + assert parent_id is None or comment_id_is_valid(parent_id) + # non-strict removes reference restrictions + else: + assert claim_id or parent_id + if claim_id: + assert claim_id_is_valid(claim_id) + else: + assert comment_id_is_valid(parent_id) + + except AssertionError: + return False + else: + return is_valid_credential_input(**kwargs) def is_valid_credential_input(channel_id: str = None, channel_name: str = None, From 8f12d997aeae2cf2eb9041f241eebe5971fbb039 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 31 Mar 2020 13:59:37 -0400 Subject: [PATCH 26/46] json -> yaml --- src/definitions.py | 2 +- src/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/definitions.py b/src/definitions.py index 9972e13..51654e1 100644 --- a/src/definitions.py +++ b/src/definitions.py @@ -2,6 +2,6 @@ import os SRC_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(SRC_DIR) -CONFIG_FILE = os.path.join(ROOT_DIR, 'config', 'conf.json') +CONFIG_FILE = os.path.join(ROOT_DIR, 'config', 'conf.yml') LOGGING_DIR = os.path.join(ROOT_DIR, 'logs') DATABASE_DIR = os.path.join(ROOT_DIR, 'database') diff --git a/src/main.py b/src/main.py index 124c7d6..817dac4 100644 --- a/src/main.py +++ b/src/main.py @@ -79,7 +79,7 @@ def setup_logging_from_config(conf: dict): def get_config(filepath): with open(filepath, 'r') as cfile: - config = yaml.load(cfile, Loader=yaml.FullLoader) + config = yaml.load(cfile, Loader=yaml.Loader) return config From c852697c947f3efb0918bd0d0cdad82465ae9272 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Wed, 1 Apr 2020 18:12:46 -0400 Subject: [PATCH 27/46] reimplements `abandon` method using ORM; no longer require `channel_id` param --- src/database/models.py | 9 ++++++--- src/database/writes.py | 4 ++++ src/server/handles.py | 43 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/database/models.py b/src/database/models.py index f999f34..73f8f17 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -113,9 +113,12 @@ def comment_list(claim_id: str = None, parent_id: str = None, def get_comment(comment_id: str) -> dict: - return (comment_list(expressions=(Comment.comment_id == comment_id), page_size=1) - .get('items') - .pop()) + try: + comment = comment_list(expressions=(Comment.comment_id == comment_id), page_size=1).get('items').pop() + except IndexError: + raise ValueError(f'Comment does not exist with id {comment_id}') + else: + return comment def create_comment_id(comment: str, channel_id: str, timestamp: int): diff --git a/src/database/writes.py b/src/database/writes.py index 9f13f3f..61caa78 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -84,13 +84,17 @@ async def hide_comments(app, pieces: list) -> list: # TODO: Amortize this process claims = {} comments_to_hide = [] + # go through a list of dict objects for p in pieces: + # maps the comment_id from the piece to a claim_id claim_id = comment_cids[p['comment_id']] + # resolve the claim from its id if claim_id not in claims: claim = await get_claim_from_id(app, claim_id) if claim: claims[claim_id] = claim + # get the claim's signing channel, then use it to validate the hidden comment channel = claims[claim_id].get('signing_channel') if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']): comments_to_hide.append(p) diff --git a/src/server/handles.py b/src/server/handles.py index bcc2fa6..373fc74 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -5,6 +5,7 @@ import typing from aiohttp import web from aiojobs.aiohttp import atomic +from peewee import DoesNotExist from src.server.validation import validate_signature_from_claim from src.misc import clean_input_params, get_claim_from_id @@ -110,9 +111,45 @@ def get_channel_from_comment_id(app, comment_id: str) -> dict: return results['items'].pop() -async def handle_abandon_comment(app, params): - # return {'abandoned': await abandon_comment(app, **params)} - raise NotImplementedError +async def handle_abandon_comment( + app: web.Application, + comment_id: str, + signature: str, + signing_ts: str, + **kwargs, +) -> dict: + comment = get_comment(comment_id) + try: + channel = await get_claim_from_id(app, comment['channel_id']) + except DoesNotExist: + raise ValueError('Could not find a channel associated with the given comment') + else: + if not validate_signature_from_claim(channel, signature, signing_ts, comment_id): + raise ValueError('Abandon signature could not be validated') + + with app['db'].atomic(): + return { + 'abandoned': delete_comment(comment_id) + } + + +async def handle_hide_comments(app: web.Application, pieces: list, hide: bool = True) -> dict: + # let's get all the distinct claim_ids from the list of comment_ids + pieces_by_id = {p['comment_id']: p for p in pieces} + comment_ids = list(pieces_by_id.keys()) + comments = (Comment + .select(Comment.comment_id, Comment.claim_id) + .where(Comment.comment_id.in_(comment_ids)) + .tuples()) + + # resolve the claims and map them to their corresponding comment_ids + claims = {} + for comment_id, claim_id in comments: + try: + # try and resolve the claim, if fails then we mark it as null + # and remove the associated comment from the pieces + if claim_id not in claims: + claims[claim_id] = await get_claim_from_id(app, claim_id) async def handle_hide_comments(app, pieces: list = None, claim_id: str = None) -> dict: From 08060a71d3b1a183536c3d38190e6d10fdfc0ac3 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Wed, 1 Apr 2020 18:13:28 -0400 Subject: [PATCH 28/46] Implements `hide_comments` method using ORM --- src/server/handles.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/server/handles.py b/src/server/handles.py index 373fc74..6857908 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -151,11 +151,37 @@ async def handle_hide_comments(app: web.Application, pieces: list, hide: bool = if claim_id not in claims: claims[claim_id] = await get_claim_from_id(app, claim_id) + # try to get a public key to validate + if claims[claim_id] is None or 'signing_channel' not in claims[claim_id]: + raise ValueError(f'could not get signing channel from claim_id: {claim_id}') -async def handle_hide_comments(app, pieces: list = None, claim_id: str = None) -> dict: + # try to validate signature + else: + channel = claims[claim_id]['signing_channel'] + piece = pieces_by_id[comment_id] + is_valid_signature = validate_signature_from_claim( + claim=channel, + signature=piece['signature'], + signing_ts=piece['signing_ts'], + data=piece['comment_id'] + ) + if not is_valid_signature: + raise ValueError(f'could not validate signature on comment_id: {comment_id}') - # return {'hidden': await hide_comments(app, **params)} - raise NotImplementedError + except ValueError: + # remove the piece from being hidden + pieces_by_id.pop(comment_id) + + # remaining items in pieces_by_id have been able to successfully validate + with app['db'].atomic(): + set_hidden_flag(list(pieces_by_id.keys()), hidden=hide) + + query = Comment.select().where(Comment.comment_id.in_(comment_ids)).objects() + result = { + 'hidden': [c.comment_id for c in query if c.is_hidden], + 'visible': [c.comment_id for c in query if not c.is_hidden], + } + return result async def handle_edit_comment(app, comment: str = None, comment_id: str = None, From 20f9ccc8c53fb11f41166a29c39e56c5dedaddaf Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Wed, 1 Apr 2020 18:49:03 -0400 Subject: [PATCH 29/46] Moves to using `CharField` in place of `TextField` --- src/database/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/database/models.py b/src/database/models.py index 73f8f17..475d167 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -13,15 +13,15 @@ from src.misc import clean class Channel(Model): - claim_id = TextField(column_name='ClaimId', primary_key=True) - name = TextField(column_name='Name') + claim_id = CharField(column_name='ClaimId', primary_key=True, max_length=40) + name = CharField(column_name='Name', max_length=256) class Meta: table_name = 'CHANNEL' class Comment(Model): - comment = TextField(column_name='Body') + comment = CharField(column_name='Body', max_length=2000) channel = ForeignKeyField( backref='comments', column_name='ChannelId', @@ -29,9 +29,9 @@ class Comment(Model): model=Channel, null=True ) - comment_id = TextField(column_name='CommentId', primary_key=True) + comment_id = CharField(column_name='CommentId', primary_key=True, max_length=64) is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")]) - claim_id = TextField(column_name='LbryClaimId') + claim_id = CharField(max_length=40, column_name='LbryClaimId') parent = ForeignKeyField( column_name='ParentId', field='comment_id', @@ -39,7 +39,7 @@ class Comment(Model): null=True, backref='replies' ) - signature = TextField(column_name='Signature', null=True, unique=True) + signature = CharField(max_length=128, column_name='Signature', null=True, unique=True) signing_ts = TextField(column_name='SigningTs', null=True) timestamp = IntegerField(column_name='Timestamp') From 3b6b05200086ea0fc44c4f16619ce5636dc25faf Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Wed, 1 Apr 2020 18:49:37 -0400 Subject: [PATCH 30/46] Removes incorrect method for `get_channel_from_comment_id` --- src/server/handles.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/server/handles.py b/src/server/handles.py index 6857908..10d2dda 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -101,16 +101,6 @@ def handle_get_claim_hidden_comments( ) -def get_channel_from_comment_id(app, comment_id: str) -> dict: - results = comment_list( - expressions=(Comment.comment_id == comment_id), - select_fields=['channel_name', 'channel_id', 'channel_url'], - page_size=1 - ) - # todo: make the return type here consistent - return results['items'].pop() - - async def handle_abandon_comment( app: web.Application, comment_id: str, @@ -218,7 +208,7 @@ METHODS = { 'get_claim_hidden_comments': handle_get_claim_hidden_comments, # this gets used 'get_comment_ids': handle_get_comment_ids, 'get_comments_by_id': handle_get_comments_by_id, # this gets used - 'get_channel_from_comment_id': get_channel_from_comment_id, # this gets used + 'get_channel_from_comment_id': handle_get_channel_from_comment_id, # this gets used 'create_comment': handle_create_comment, # this gets used 'delete_comment': handle_abandon_comment, 'abandon_comment': handle_abandon_comment, # this gets used From 84cafd643fad0b569d2a3f952cd3e0d441c3f673 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Wed, 1 Apr 2020 18:50:09 -0400 Subject: [PATCH 31/46] replace oracle driver with `pymysql` driver --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c8755cf..f7a397f 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( data_files=[('config', ['config/conf.json',])], include_package_data=True, install_requires=[ - 'mysql-connector-python', + 'pymysql', 'pyyaml', 'Faker>=1.0.7', 'asyncio', From c27baa89fe79898b6b5b46ded2542dfbf3422abe Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Wed, 1 Apr 2020 18:52:36 -0400 Subject: [PATCH 32/46] rate limit comments --- src/database/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/models.py b/src/database/models.py index 475d167..9c40586 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -124,7 +124,7 @@ def get_comment(comment_id: str) -> dict: def create_comment_id(comment: str, channel_id: str, timestamp: int): # We convert the timestamp from seconds into minutes # to prevent spammers from commenting the same BS everywhere. - nearest_minute = str(math.floor(timestamp)) + nearest_minute = str(math.floor(timestamp / 60)) # don't use claim_id for the comment_id anymore so comments # are not unique to just one claim From fd05ea1145bed5389d962350b39e9b1ba21561d3 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 3 Apr 2020 15:33:23 -0400 Subject: [PATCH 33/46] Update travis config & add generic yaml config --- .travis.yml | 23 ++++++++++++++++++++++- config/conf.yml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 config/conf.yml diff --git a/.travis.yml b/.travis.yml index ac2c250..fe81b23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,28 @@ sudo: required language: python dist: xenial -python: 3.7 +python: 3.8 + +# for docker-compose +services: + - docker + +# to avoid "well it works on my computer" moments +env: + - DOCKER_COMPOSE_VERSION=1.25.4 + +before_install: + # ensure docker-compose version is as specified above + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin + # refresh docker images + - sudo apt-get update + +before_script: + - docker-compose up -d + jobs: include: diff --git a/config/conf.yml b/config/conf.yml new file mode 100644 index 0000000..6d90446 --- /dev/null +++ b/config/conf.yml @@ -0,0 +1,30 @@ +--- +# for running local-tests without using MySQL for now +testing: + database: sqlite + file: comments.db + pragmas: + journal_mode: wal + cache_size: 64000 + foreign_keys: 0 + ignore_check_constraints: 1 + synchronous: 0 + +# actual database should be running MySQL +production: + database: mysql + name: lbry + user: lbry + password: lbry + host: localhost + port: 3306 + +mode: production +logging: + format: "%(asctime)s | %(levelname)s | %(name)s | %(module)s.%(funcName)s:%(lineno)d + | %(message)s" + aiohttp_format: "%(asctime)s | %(levelname)s | %(name)s | %(message)s" + datefmt: "%Y-%m-%d %H:%M:%S" +host: localhost +port: 5921 +lbrynet: http://localhost:5279 \ No newline at end of file From a84e0e0f84d344b512f63a10db4db033d217ccbd Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 3 Apr 2020 15:38:29 -0400 Subject: [PATCH 34/46] Add docker-compose.yml --- docker-compose.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3faf5f2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3.7" +services: + ########### + ## MySQL ## + ########### + mysql: + image: mysql/mysql-server:5.7.27 + restart: "no" + ports: + - "3306:3306" + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - MYSQL_DATABASE=lbry + - MYSQL_USER=lbry + - MYSQL_PASSWORD=lbry + - MYSQL_LOG_CONSOLE=true + ############# + ## Adminer ## + ############# + adminer: + image: adminer + restart: always + ports: + - 8080:8080 From 38e9af24ecf5e31b9c76322197ee87e67110b202 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 3 Apr 2020 16:07:23 -0400 Subject: [PATCH 35/46] Update .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c7e09d7..54e4315 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -config/conf.json +config/conf.yml +docker-compose.yml + From 75c8f82072b963f4fe7263013f3e69f87c983967 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Fri, 3 Apr 2020 16:40:05 -0400 Subject: [PATCH 36/46] Add todos --- src/server/handles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/handles.py b/src/server/handles.py index 10d2dda..1c56663 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -187,6 +187,7 @@ async def handle_edit_comment(app, comment: str = None, comment_id: str = None, return get_comment(comment_id) +# TODO: retrieve stake amounts for each channel & store in db def handle_create_comment(app, comment: str = None, claim_id: str = None, parent_id: str = None, channel_id: str = None, channel_name: str = None, signature: str = None, signing_ts: str = None) -> dict: From d25e03d853a585e5f9ac816f4ebdc020649a2636 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 6 Apr 2020 19:13:12 -0400 Subject: [PATCH 37/46] database name gets set to `social` --- config/conf.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/conf.yml b/config/conf.yml index 6d90446..b311ef1 100644 --- a/config/conf.yml +++ b/config/conf.yml @@ -13,7 +13,7 @@ testing: # actual database should be running MySQL production: database: mysql - name: lbry + name: social user: lbry password: lbry host: localhost diff --git a/docker-compose.yml b/docker-compose.yml index 3faf5f2..41530f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - "3306:3306" environment: - MYSQL_ALLOW_EMPTY_PASSWORD=true - - MYSQL_DATABASE=lbry + - MYSQL_DATABASE=social - MYSQL_USER=lbry - MYSQL_PASSWORD=lbry - MYSQL_LOG_CONSOLE=true From be45a70c362a4bc55bdca143769c044c3a6a2afe Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 6 Apr 2020 19:15:54 -0400 Subject: [PATCH 38/46] sets column names to be lowercase, uses utf8mb4 charset, utf8mb4_unicode_ci collation --- config/conf.yml | 1 + database/default_after.sql | 48 ++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 1 + src/database/models.py | 16 ++++++++----- src/misc.py | 2 +- src/server/app.py | 1 + 6 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 database/default_after.sql diff --git a/config/conf.yml b/config/conf.yml index b311ef1..fc22322 100644 --- a/config/conf.yml +++ b/config/conf.yml @@ -12,6 +12,7 @@ testing: # actual database should be running MySQL production: + charset: utf8mb4 database: mysql name: social user: lbry diff --git a/database/default_after.sql b/database/default_after.sql new file mode 100644 index 0000000..1ee19f4 --- /dev/null +++ b/database/default_after.sql @@ -0,0 +1,48 @@ +USE `social`; +ALTER DATABASE `social` + DEFAULT CHARACTER SET utf8mb4 + DEFAULT COLLATE utf8mb4_unicode_ci; + +DROP TABLE IF EXISTS `CHANNEL`; +CREATE TABLE `CHANNEL` ( + `claimid` VARCHAR(40) NOT NULL, + -- i cant tell if max name length is 255 or 256 + `name` VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, + CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`) + ) +CHARACTER SET utf8mb4 +COLLATE utf8mb4_unicode_ci; + +DROP TABLE IF EXISTS `COMMENT`; +CREATE TABLE `COMMENT` ( + `commentid` VARCHAR(64) NOT NULL, + `lbryclaimid` VARCHAR(40) NOT NULL, + `channelid` VARCHAR(40) DEFAULT NULL, + `body` VARCHAR(5000) + CHARACTER SET utf8mb4 + COLLATE utf8mb4_unicode_ci + NOT NULL, + `parentid` VARCHAR(64) DEFAULT NULL, + `signature` VARCHAR(128) DEFAULT NULL, + `signingts` VARCHAR(22) DEFAULT NULL, + + `timestamp` INTEGER NOT NULL, + -- there's no way that the timestamp will ever reach 22 characters + `ishidden` BOOLEAN DEFAULT FALSE, + CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`), + CONSTRAINT `comment_signature_sk` UNIQUE (`signature`), + CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`) + ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`) + ON UPDATE CASCADE ON DELETE CASCADE, -- setting null implies comment is top level + CONSTRAINT `channel_signature` + CHECK ( `signature` IS NOT NULL AND `signingts` IS NOT NULL) + ) +CHARACTER SET utf8mb4 +COLLATE utf8mb4_unicode_ci; + +CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`); +CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`); + + + diff --git a/docker-compose.yml b/docker-compose.yml index 41530f7..e5f9099 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: mysql: image: mysql/mysql-server:5.7.27 restart: "no" + command: --character_set_server=utf8mb4 --max_allowed_packet=1073741824 ports: - "3306:3306" environment: diff --git a/src/database/models.py b/src/database/models.py index 9c40586..9e72d33 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -13,24 +13,28 @@ from src.misc import clean class Channel(Model): - claim_id = CharField(column_name='ClaimId', primary_key=True, max_length=40) - name = CharField(column_name='Name', max_length=256) + claim_id = CharField(column_name='claimid', primary_key=True, max_length=40) + name = CharField(column_name='name', max_length=256) class Meta: table_name = 'CHANNEL' class Comment(Model): - comment = CharField(column_name='Body', max_length=2000) + comment = CharField( + column_name='body', + max_length=5000, + + ) channel = ForeignKeyField( backref='comments', - column_name='ChannelId', + column_name='channelid', field='claim_id', model=Channel, null=True ) - comment_id = CharField(column_name='CommentId', primary_key=True, max_length=64) - is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")]) + comment_id = CharField(column_name='commentid', primary_key=True, max_length=64) + is_hidden = BooleanField(column_name='ishidden', constraints=[SQL("DEFAULT 0")]) claim_id = CharField(max_length=40, column_name='LbryClaimId') parent = ForeignKeyField( column_name='ParentId', diff --git a/src/misc.py b/src/misc.py index e9d5be8..7c3d4bf 100644 --- a/src/misc.py +++ b/src/misc.py @@ -16,7 +16,7 @@ async def get_claim_from_id(app, claim_id, **kwargs): def clean_input_params(kwargs: dict): for k, v in kwargs.items(): - if type(v) is str and k is not 'comment': + if type(v) is str and k != 'comment': kwargs[k] = v.strip() if k in ID_LIST: kwargs[k] = v.lower() diff --git a/src/server/app.py b/src/server/app.py index 389083e..37e89da 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -28,6 +28,7 @@ def setup_database(app): host=config[mode]['host'], password=config[mode]['password'], port=config[mode]['port'], + charset=config[mode]['charset'], ) elif config[mode]['database'] == 'sqlite': app['db'] = SqliteDatabase( From c7e8d274f7555786100a576c1559bd42dac4af8f Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Mon, 6 Apr 2020 19:26:22 -0400 Subject: [PATCH 39/46] update travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fe81b23..fe9a4de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,5 @@ jobs: name: "Unit Tests" install: - pip install -e . - - mkdir database script: - python -m unittest From 7b7e6c66acb0dd18b5bf17294e0a3736e497fe17 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 7 Apr 2020 15:04:50 -0400 Subject: [PATCH 40/46] `VARCHAR` -> `CHAR` for all ID fields, constraints moved out of table definition --- database/default_after.sql | 39 ++++++++++++++++++++++---------------- src/database/models.py | 20 ++++++++----------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/database/default_after.sql b/database/default_after.sql index 1ee19f4..e6a606b 100644 --- a/database/default_after.sql +++ b/database/default_after.sql @@ -6,8 +6,7 @@ ALTER DATABASE `social` DROP TABLE IF EXISTS `CHANNEL`; CREATE TABLE `CHANNEL` ( `claimid` VARCHAR(40) NOT NULL, - -- i cant tell if max name length is 255 or 256 - `name` VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, + `name` CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`) ) CHARACTER SET utf8mb4 @@ -15,34 +14,42 @@ COLLATE utf8mb4_unicode_ci; DROP TABLE IF EXISTS `COMMENT`; CREATE TABLE `COMMENT` ( - `commentid` VARCHAR(64) NOT NULL, - `lbryclaimid` VARCHAR(40) NOT NULL, - `channelid` VARCHAR(40) DEFAULT NULL, - `body` VARCHAR(5000) + -- should be changed to CHAR(64) + `commentid` CHAR(64) NOT NULL, + -- should be changed to CHAR(40) + `lbryclaimid` CHAR(40) NOT NULL, + -- can be null, so idk if this should be char(40) + `channelid` CHAR(40) DEFAULT NULL, + `body` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, - `parentid` VARCHAR(64) DEFAULT NULL, - `signature` VARCHAR(128) DEFAULT NULL, + `parentid` CHAR(64) DEFAULT NULL, + `signature` CHAR(128) DEFAULT NULL, + -- 22 chars long is prolly enough `signingts` VARCHAR(22) DEFAULT NULL, `timestamp` INTEGER NOT NULL, -- there's no way that the timestamp will ever reach 22 characters `ishidden` BOOLEAN DEFAULT FALSE, - CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`), - CONSTRAINT `comment_signature_sk` UNIQUE (`signature`), - CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`) - ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`) - ON UPDATE CASCADE ON DELETE CASCADE, -- setting null implies comment is top level - CONSTRAINT `channel_signature` - CHECK ( `signature` IS NOT NULL AND `signingts` IS NOT NULL) + CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`) + -- setting null implies comment is top level ) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +ALTER TABLE COMMENT + ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`) + ON DELETE CASCADE ON UPDATE CASCADE, + ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`) + ON UPDATE CASCADE ON DELETE CASCADE +; + CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`); CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`); +ALTER TABLE COMMENT ADD CONSTRAINT UNIQUE (`signature`, `channelid`); + diff --git a/src/database/models.py b/src/database/models.py index 9e72d33..b35f122 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -13,19 +13,15 @@ from src.misc import clean class Channel(Model): - claim_id = CharField(column_name='claimid', primary_key=True, max_length=40) - name = CharField(column_name='name', max_length=256) + claim_id = FixedCharField(column_name='claimid', primary_key=True, max_length=40) + name = CharField(column_name='name', max_length=255) class Meta: table_name = 'CHANNEL' class Comment(Model): - comment = CharField( - column_name='body', - max_length=5000, - - ) + comment = TextField(column_name='body') channel = ForeignKeyField( backref='comments', column_name='channelid', @@ -33,9 +29,9 @@ class Comment(Model): model=Channel, null=True ) - comment_id = CharField(column_name='commentid', primary_key=True, max_length=64) + comment_id = FixedCharField(column_name='commentid', primary_key=True, max_length=64) is_hidden = BooleanField(column_name='ishidden', constraints=[SQL("DEFAULT 0")]) - claim_id = CharField(max_length=40, column_name='LbryClaimId') + claim_id = FixedCharField(max_length=40, column_name='lbryclaimid') parent = ForeignKeyField( column_name='ParentId', field='comment_id', @@ -43,9 +39,9 @@ class Comment(Model): null=True, backref='replies' ) - signature = CharField(max_length=128, column_name='Signature', null=True, unique=True) - signing_ts = TextField(column_name='SigningTs', null=True) - timestamp = IntegerField(column_name='Timestamp') + signature = FixedCharField(max_length=128, column_name='signature', null=True, unique=True) + signing_ts = TextField(column_name='signingts', null=True) + timestamp = IntegerField(column_name='timestamp') class Meta: table_name = 'COMMENT' From 0817b700830316429c5ac53fca47c1ef635f98d8 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 7 Apr 2020 16:08:29 -0400 Subject: [PATCH 41/46] Update README.md --- README.md | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 471f552..cdeef1d 100644 --- a/README.md +++ b/README.md @@ -3,34 +3,36 @@ [](https://travis-ci.com/lbryio/comment-server) [](https://codeclimate.com/github/lbryio/comment-server/maintainability) -This is the code for the LBRY Comment Server. -Fork it, run it, set it on fire. Up to you. - ## Before Installing -Comment Deletion requires having the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) +Install the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) in order to validate & properly delete comments. - - + ## Installation #### Installing the server: ```bash -$ git clone https://github.com/osilkin98/comment-server +$ git clone https://github.com/lbryio/comment-server $ cd comment-server # create a virtual environment -$ virtualenv --python=python3 venv +$ virtualenv --python=python3.8 venv # Enter the virtual environment $ source venv/bin/activate -# install the Server as a Executable Target -(venv) $ python setup.py develop +# Install required dependencies +(venv) $ pip install -e . + +# Run the server +(venv) $ python src/main.py \ + --port=5921 \ # use a different port besides the default + --config=conf.yml \ # provide a custom config file + & \ # detach and run the service in the background ``` ### Installing the systemd Service Monitor @@ -70,16 +72,11 @@ To Test the database, simply run: There are basic tests to run against the server, though they require that there is a server instance running, though the database - chosen may have to be edited in `config/conf.json`. + chosen may have to be edited in `config/conf.yml`. Additionally there are HTTP requests that can be send with whatever software you choose to test the integrity of the comment server. -## Schema - - - - ## Contributing Contributions are welcome, verbosity is encouraged. Please be considerate From 3b91279cc73858d73f256df43994d2f992b3e7ab Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 7 Apr 2020 16:16:15 -0400 Subject: [PATCH 42/46] Moves schema from SQLite3 to MySQL --- database/default_after.sql | 55 ---------------- src/database/comments_ddl.sql | 114 +++++++++++++--------------------- src/database/schema.py | 76 ----------------------- 3 files changed, 44 insertions(+), 201 deletions(-) delete mode 100644 database/default_after.sql delete mode 100644 src/database/schema.py diff --git a/database/default_after.sql b/database/default_after.sql deleted file mode 100644 index e6a606b..0000000 --- a/database/default_after.sql +++ /dev/null @@ -1,55 +0,0 @@ -USE `social`; -ALTER DATABASE `social` - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; - -DROP TABLE IF EXISTS `CHANNEL`; -CREATE TABLE `CHANNEL` ( - `claimid` VARCHAR(40) NOT NULL, - `name` CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, - CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`) - ) -CHARACTER SET utf8mb4 -COLLATE utf8mb4_unicode_ci; - -DROP TABLE IF EXISTS `COMMENT`; -CREATE TABLE `COMMENT` ( - -- should be changed to CHAR(64) - `commentid` CHAR(64) NOT NULL, - -- should be changed to CHAR(40) - `lbryclaimid` CHAR(40) NOT NULL, - -- can be null, so idk if this should be char(40) - `channelid` CHAR(40) DEFAULT NULL, - `body` TEXT - CHARACTER SET utf8mb4 - COLLATE utf8mb4_unicode_ci - NOT NULL, - `parentid` CHAR(64) DEFAULT NULL, - `signature` CHAR(128) DEFAULT NULL, - -- 22 chars long is prolly enough - `signingts` VARCHAR(22) DEFAULT NULL, - - `timestamp` INTEGER NOT NULL, - -- there's no way that the timestamp will ever reach 22 characters - `ishidden` BOOLEAN DEFAULT FALSE, - CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`) - -- setting null implies comment is top level - ) -CHARACTER SET utf8mb4 -COLLATE utf8mb4_unicode_ci; - - -ALTER TABLE COMMENT - ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`) - ON DELETE CASCADE ON UPDATE CASCADE, - ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`) - ON UPDATE CASCADE ON DELETE CASCADE -; - -CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`); -CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`); - - -ALTER TABLE COMMENT ADD CONSTRAINT UNIQUE (`signature`, `channelid`); - - diff --git a/src/database/comments_ddl.sql b/src/database/comments_ddl.sql index 2dfc19d..f488406 100644 --- a/src/database/comments_ddl.sql +++ b/src/database/comments_ddl.sql @@ -1,76 +1,50 @@ -PRAGMA FOREIGN_KEYS = ON; +USE `social`; +ALTER DATABASE `social` + DEFAULT CHARACTER SET utf8mb4 + DEFAULT COLLATE utf8mb4_unicode_ci; --- Although I know this file is unnecessary, I like keeping it around. +DROP TABLE IF EXISTS `CHANNEL`; +CREATE TABLE `CHANNEL` ( + `claimid` VARCHAR(40) NOT NULL, + `name` CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, + CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`) + ) +CHARACTER SET utf8mb4 +COLLATE utf8mb4_unicode_ci; --- I'm not gonna remove it. +DROP TABLE IF EXISTS `COMMENT`; +CREATE TABLE `COMMENT` ( + -- should be changed to CHAR(64) + `commentid` CHAR(64) NOT NULL, + -- should be changed to CHAR(40) + `lbryclaimid` CHAR(40) NOT NULL, + -- can be null, so idk if this should be char(40) + `channelid` CHAR(40) DEFAULT NULL, + `body` TEXT + CHARACTER SET utf8mb4 + COLLATE utf8mb4_unicode_ci + NOT NULL, + `parentid` CHAR(64) DEFAULT NULL, + `signature` CHAR(128) DEFAULT NULL, + -- 22 chars long is prolly enough + `signingts` VARCHAR(22) DEFAULT NULL, --- tables -CREATE TABLE IF NOT EXISTS COMMENT -( - CommentId TEXT NOT NULL, - LbryClaimId TEXT NOT NULL, - ChannelId TEXT DEFAULT NULL, - Body TEXT NOT NULL, - ParentId TEXT DEFAULT NULL, - Signature TEXT DEFAULT NULL, - Timestamp INTEGER NOT NULL, - SigningTs TEXT DEFAULT NULL, - IsHidden BOOLEAN NOT NULL DEFAULT FALSE, - CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, - CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, - CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) - ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId) - ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level -); - --- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE); --- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL; - --- DROP TABLE IF EXISTS CHANNEL; -CREATE TABLE IF NOT EXISTS CHANNEL -( - ClaimId TEXT NOT NULL, - Name TEXT NOT NULL, - CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId) - ON CONFLICT IGNORE -); + `timestamp` INTEGER NOT NULL, + -- there's no way that the timestamp will ever reach 22 characters + `ishidden` BOOLEAN DEFAULT FALSE, + CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`) + -- setting null implies comment is top level + ) +CHARACTER SET utf8mb4 +COLLATE utf8mb4_unicode_ci; --- indexes --- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX; --- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); +ALTER TABLE COMMENT + ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`) + ON DELETE CASCADE ON UPDATE CASCADE, + ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`) + ON UPDATE CASCADE ON DELETE CASCADE +; --- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); - --- VIEWS -CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS -SELECT C.CommentId AS comment_id, - C.Body AS comment, - C.LbryClaimId AS claim_id, - C.Timestamp AS timestamp, - CHAN.Name AS channel_name, - CHAN.ClaimId AS channel_id, - ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, - C.Signature AS signature, - C.SigningTs AS signing_ts, - C.ParentId AS parent_id, - C.IsHidden AS is_hidden -FROM COMMENT AS C - LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId -ORDER BY C.Timestamp DESC; - - -DROP VIEW IF EXISTS COMMENT_REPLIES; -CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS -SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body -FROM COMMENT AS OG - JOIN COMMENT AS PARENT - ON OG.ParentId = PARENT.CommentId - JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId - JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId -ORDER BY OG.Timestamp; - --- this is the default channel for anyone who wants to publish anonymously --- INSERT INTO CHANNEL --- VALUES ('9cb713f01bf247a0e03170b5ed00d5161340c486', '@Anonymous'); +CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`); +CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`); diff --git a/src/database/schema.py b/src/database/schema.py deleted file mode 100644 index c75681b..0000000 --- a/src/database/schema.py +++ /dev/null @@ -1,76 +0,0 @@ -PRAGMAS = """ - PRAGMA FOREIGN_KEYS = ON; -""" - -CREATE_COMMENT_TABLE = """ - CREATE TABLE IF NOT EXISTS COMMENT ( - CommentId TEXT NOT NULL, - LbryClaimId TEXT NOT NULL, - ChannelId TEXT DEFAULT NULL, - Body TEXT NOT NULL, - ParentId TEXT DEFAULT NULL, - Signature TEXT DEFAULT NULL, - Timestamp INTEGER NOT NULL, - SigningTs TEXT DEFAULT NULL, - IsHidden BOOLEAN NOT NULL DEFAULT 0, - CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, - CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, - CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) - ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId) - ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level - ); -""" - -CREATE_COMMENT_INDEXES = """ - CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); - CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); -""" - -CREATE_CHANNEL_TABLE = """ - CREATE TABLE IF NOT EXISTS CHANNEL ( - ClaimId TEXT NOT NULL, - Name TEXT NOT NULL, - CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId) - ON CONFLICT IGNORE - ); -""" - -CREATE_COMMENTS_ON_CLAIMS_VIEW = """ - CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT - C.CommentId AS comment_id, - C.Body AS comment, - C.LbryClaimId AS claim_id, - C.Timestamp AS timestamp, - CHAN.Name AS channel_name, - CHAN.ClaimId AS channel_id, - ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, - C.Signature AS signature, - C.SigningTs AS signing_ts, - C.ParentId AS parent_id, - C.IsHidden AS is_hidden - FROM COMMENT AS C - LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId - ORDER BY C.Timestamp DESC; -""" - -# not being used right now but should be kept around when Tom finally asks for replies -CREATE_COMMENT_REPLIES_VIEW = """ -CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS -SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body -FROM COMMENT AS OG - JOIN COMMENT AS PARENT - ON OG.ParentId = PARENT.CommentId - JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId - JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId -ORDER BY OG.Timestamp; -""" - -CREATE_TABLES_QUERY = ( - PRAGMAS + - CREATE_COMMENT_TABLE + - CREATE_COMMENT_INDEXES + - CREATE_CHANNEL_TABLE + - CREATE_COMMENTS_ON_CLAIMS_VIEW + - CREATE_COMMENT_REPLIES_VIEW -) From b4377b2f54ae06403cb978f5084e0b271de4be6c Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Tue, 7 Apr 2020 16:16:34 -0400 Subject: [PATCH 43/46] Cleans up & adds todos for future --- scripts/stress_test.py | 84 ------------------------------------- scripts/valid_signatures.py | 3 +- src/database/writes.py | 1 + src/main.py | 1 - 4 files changed, 3 insertions(+), 86 deletions(-) delete mode 100644 scripts/stress_test.py diff --git a/scripts/stress_test.py b/scripts/stress_test.py deleted file mode 100644 index 6022396..0000000 --- a/scripts/stress_test.py +++ /dev/null @@ -1,84 +0,0 @@ -import sqlite3 -import time - -import faker -from faker.providers import misc - -fake = faker.Faker() -fake.add_provider(misc) - - -if __name__ == '__main__': - song_time = """One, two, three! -My baby don't mess around -'Cause she loves me so -This I know fo sho! -But does she really wanna -But can't stand to see me walk out tha door -Don't try to fight the feeling -Because the thought alone is killin' me right now -Thank God for Mom and Dad -For sticking to together -Like we don't know how -Hey ya! Hey ya! -Hey ya! Hey ya! -Hey ya! Hey ya! -Hey ya! Hey ya! -You think you've got it -Oh, you think you've got it -But got it just don't get it when there's nothin' at all -We get together -Oh, we get together -But separate's always better when there's feelings involved -Know what they say -its -Nothing lasts forever! -Then what makes it, then what makes it -Then what makes it, then what makes it -Then what makes love the exception? -So why, oh, why, oh -Why, oh, why, oh, why, oh -Are we still in denial when we know we're not happy here -Hey ya! (y'all don't want to here me, ya just want to dance) Hey ya! -Don't want to meet your daddy (oh ohh), just want you in my caddy (oh ohh) -Hey ya! (oh, oh!) Hey ya! (oh, oh!) -Don't want to meet your momma, just want to make you cum-a (oh, oh!) -I'm (oh, oh) I'm (oh, oh) I'm just being honest! (oh, oh) -I'm just being honest! -Hey! alright now! alright now, fellas! -Yea? -Now, what cooler than being cool? -Ice cold! -I can't hear ya! I say what's, what's cooler than being cool? -Ice cold! -Alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright! -Okay, now ladies! -Yea? -Now we gonna break this thang down for just a few seconds -Now don't have me break this thang down for nothin' -I want to see you on your badest behavior! -Lend me some sugar, I am your neighbor! -Ah! Here we go now, -Shake it, shake it, shake it, shake it, shake it -Shake it, shake it, shake it, shake it -Shake it like a Polaroid picture! Hey ya! -Shake it, shake it, shake it, shake it, shake it -Shake it, shake it, shake it, suga! -Shake it like a Polaroid picture! -Now all the Beyonce's, and Lucy Lu's, and baby dolls -Get on tha floor get on tha floor! -Shake it like a Polaroid picture! -Oh, you! oh, you! -Hey ya!(oh, oh) Hey ya!(oh, oh) -Hey ya!(oh, oh) Hey ya!(oh, oh) -Hey ya!(oh, oh) Hey ya!(oh, oh)""" - - song = song_time.split('\n') - claim_id = '2aa106927b733e2602ffb565efaccc78c2ed89df' - run_len = [(fake.sha256(), song_time, claim_id, str(int(time.time()))) for k in range(5000)] - - conn = sqlite3.connect('database/default_test.db') - with conn: - curs = conn.executemany(""" - INSERT INTO COMMENT(CommentId, Body, LbryClaimId, Timestamp) VALUES (?, ?, ?, ?) - """, run_len) - print(f'rows changed: {curs.rowcount}') diff --git a/scripts/valid_signatures.py b/scripts/valid_signatures.py index 19fc878..4b7c641 100644 --- a/scripts/valid_signatures.py +++ b/scripts/valid_signatures.py @@ -2,11 +2,12 @@ import binascii import logging import hashlib import json +# todo: remove sqlite3 as a dependency import sqlite3 import asyncio import aiohttp -from server.validation import is_signature_valid, get_encoded_signature +from src.server.validation import is_signature_valid, get_encoded_signature logger = logging.getLogger(__name__) diff --git a/src/database/writes.py b/src/database/writes.py index 61caa78..b86e31b 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -1,3 +1,4 @@ +# TODO: scrap notification routines from these files & supply them in handles import logging import sqlite3 from asyncio import coroutine diff --git a/src/main.py b/src/main.py index 817dac4..d4b5a53 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,4 @@ import argparse -import json import yaml import logging import logging.config From 3cce89cbacc315a6e896e46e0a98eb3ea2b23510 Mon Sep 17 00:00:00 2001 From: Oleg Silkin <o.silkin98@gmail.com> Date: Thu, 9 Apr 2020 16:55:07 -0400 Subject: [PATCH 44/46] serializes error messages into json format --- src/server/errors.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/server/errors.py b/src/server/errors.py index 273e4bd..644ce9e 100644 --- a/src/server/errors.py +++ b/src/server/errors.py @@ -1,3 +1,5 @@ +import json + import logging import aiohttp @@ -32,8 +34,11 @@ def make_error(error, exc=None) -> dict: async def report_error(app, exc, body: dict): try: if 'slack_webhook' in app['config']: + body_dump = json.dumps(body, indent=4) + exec_name = type(exc).__name__ + exec_body = str(exc) message = { - "text": f"Got `{type(exc).__name__}`: `\n{str(exc)}`\n```{body}```" + "text": f"Got `{exec_name}`: `\n{exec_body}`\n```{body_dump}```" } async with aiohttp.ClientSession() as sesh: async with sesh.post(app['config']['slack_webhook'], json=message) as resp: From 7c26b809713dea812ce0df0b75a8f67872903fa0 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr <markbeamerjr@gmail.com> Date: Tue, 21 Jul 2020 02:35:33 -0400 Subject: [PATCH 45/46] Fix notifications --- src/server/handles.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/server/handles.py b/src/server/handles.py index 1c56663..a7fe43b 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -7,6 +7,7 @@ from aiohttp import web from aiojobs.aiohttp import atomic from peewee import DoesNotExist +from src.server.external import send_notification from src.server.validation import validate_signature_from_claim from src.misc import clean_input_params, get_claim_from_id from src.server.errors import make_error, report_error @@ -116,7 +117,7 @@ async def handle_abandon_comment( else: if not validate_signature_from_claim(channel, signature, signing_ts, comment_id): raise ValueError('Abandon signature could not be validated') - + await app['webhooks'].spawn(send_notification(app, 'DELETE', comment)) with app['db'].atomic(): return { 'abandoned': delete_comment(comment_id) @@ -184,15 +185,17 @@ async def handle_edit_comment(app, comment: str = None, comment_id: str = None, with app['db'].atomic(): if not edit_comment(comment_id, comment, signature, signing_ts): raise ValueError('Comment could not be edited') - return get_comment(comment_id) + updated_comment = get_comment(comment_id) + await app['webhooks'].spawn(send_notification(app, 'UPDATE', updated_comment)) + return updated_comment # TODO: retrieve stake amounts for each channel & store in db -def handle_create_comment(app, comment: str = None, claim_id: str = None, +async def handle_create_comment(app, comment: str = None, claim_id: str = None, parent_id: str = None, channel_id: str = None, channel_name: str = None, signature: str = None, signing_ts: str = None) -> dict: with app['db'].atomic(): - return create_comment( + comment = create_comment( comment=comment, claim_id=claim_id, parent_id=parent_id, @@ -201,6 +204,8 @@ def handle_create_comment(app, comment: str = None, claim_id: str = None, signature=signature, signing_ts=signing_ts ) + await app['webhooks'].spawn(send_notification(app, 'CREATE', comment)) + return comment METHODS = { From c2230cdefb05c904051b0567b08d89427df3bb1a Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr <markbeamerjr@gmail.com> Date: Fri, 24 Jul 2020 15:43:34 -0400 Subject: [PATCH 46/46] Send parent id and comment text to internal-apis for notifications --- src/server/external.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/external.py b/src/server/external.py index aa0e7ee..25bd634 100644 --- a/src/server/external.py +++ b/src/server/external.py @@ -37,6 +37,10 @@ def create_notification_batch(action: str, comments: List[dict]) -> List[dict]: } if comment.get('channel_id'): event['channel_id'] = comment['channel_id'] + if comment.get('parent_id'): + event['parent_id'] = comment['parent_id'] + if comment.get('comment'): + event['comment'] = comment['comment'] events.append(event) return events