From 7916d8b7ff12a749167117eadc0793d32d2276ce Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 6 Jan 2020 23:03:59 -0500
Subject: [PATCH 01/46] All comment returning methods now include `claim_id`

---
 src/database/queries.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/src/database/queries.py b/src/database/queries.py
index 16e6f0b..03815aa 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -12,14 +12,8 @@ from src.database.schema import CREATE_TABLES_QUERY
 logger = logging.getLogger(__name__)
 
 SELECT_COMMENTS_ON_CLAIMS = """
-    SELECT comment, comment_id, channel_name, channel_id, channel_url,
-        timestamp, signature, signing_ts, parent_id, is_hidden
-    FROM COMMENTS_ON_CLAIMS 
-"""
-
-SELECT_COMMENTS_ON_CLAIMS_CLAIMID = """
-    SELECT comment, comment_id, claim_id, channel_name, channel_id, channel_url,
-        timestamp, signature, signing_ts, parent_id, is_hidden
+    SELECT comment, comment_id, claim_id, timestamp, is_hidden, parent_id,
+        channel_name, channel_id, channel_url, signature, signing_ts 
     FROM COMMENTS_ON_CLAIMS 
 """
 
@@ -164,7 +158,7 @@ def insert_reply(conn: sqlite3.Connection, comment: str, parent_id: str,
 
 def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict:
     with conn:
-        curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS_CLAIMID + "WHERE comment_id = ?", (comment_id,))
+        curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,))
         thing = curry.fetchone()
         return clean(dict(thing)) if thing else None
 
@@ -199,7 +193,7 @@ def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.U
     placeholders = ', '.join('?' for _ in comment_ids)
     with conn:
         return [clean(dict(row)) for row in conn.execute(
-            SELECT_COMMENTS_ON_CLAIMS_CLAIMID + f'WHERE comment_id IN ({placeholders})',
+            SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})',
             tuple(comment_ids)
         )]
 

From 25eb4f9acdcfedc80259f53bf92f22225d35cdac Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 6 Jan 2020 23:16:46 -0500
Subject: [PATCH 02/46] Prevents claim resolve error from disrupting entire
 hide operation

---
 src/database/writes.py | 6 ++++--
 src/server/misc.py     | 5 ++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/database/writes.py b/src/database/writes.py
index e724521..0e7b17d 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -88,7 +88,10 @@ async def hide_comments(app, pieces: list) -> list:
     for p in pieces:
         claim_id = comment_cids[p['comment_id']]
         if claim_id not in claims:
-            claims[claim_id] = await get_claim_from_id(app, claim_id, no_totals=True)
+            claim = await get_claim_from_id(app, claim_id)
+            if claim:
+                claims[claim_id] = claim
+
         channel = claims[claim_id].get('signing_channel')
         if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']):
             comments_to_hide.append(p)
@@ -100,7 +103,6 @@ async def hide_comments(app, pieces: list) -> list:
             app, 'UPDATE', db.get_comments_by_id(app['reader'], comment_ids)
         )
     )
-
     await job.wait()
     return comment_ids
 
diff --git a/src/server/misc.py b/src/server/misc.py
index 593e61a..4f620e6 100644
--- a/src/server/misc.py
+++ b/src/server/misc.py
@@ -8,7 +8,10 @@ ID_LIST = {'claim_id', 'parent_id', 'comment_id', 'channel_id'}
 
 
 async def get_claim_from_id(app, claim_id, **kwargs):
-    return (await request_lbrynet(app, 'claim_search', claim_id=claim_id, **kwargs))['items'][0]
+    try:
+        return (await request_lbrynet(app, 'claim_search', claim_id=claim_id, **kwargs))['items'][0]
+    except IndexError:
+        return
 
 
 def clean_input_params(kwargs: dict):

From 6a00c7aa82332f9c4971a19a3cb0306b65c42a4d Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 19 Jan 2020 23:11:07 -0500
Subject: [PATCH 03/46] Better error logging

---
 src/server/errors.py  | 10 ++++------
 src/server/handles.py |  2 +-
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/server/errors.py b/src/server/errors.py
index c0b8af2..273e4bd 100644
--- a/src/server/errors.py
+++ b/src/server/errors.py
@@ -29,16 +29,14 @@ def make_error(error, exc=None) -> dict:
         return body
 
 
-async def report_error(app, exc, msg=''):
+async def report_error(app, exc, body: dict):
     try:
         if 'slack_webhook' in app['config']:
-            if msg:
-                msg = f'"{msg}"'
-            body = {
-                "text": f"Got `{type(exc).__name__}`: ```\n{str(exc)}```\n{msg}"
+            message = {
+                "text": f"Got `{type(exc).__name__}`: `\n{str(exc)}`\n```{body}```"
             }
             async with aiohttp.ClientSession() as sesh:
-                async with sesh.post(app['config']['slack_webhook'], json=body) as resp:
+                async with sesh.post(app['config']['slack_webhook'], json=message) as resp:
                     await resp.wait_for_close()
 
     except Exception:
diff --git a/src/server/handles.py b/src/server/handles.py
index 63503f3..6f629a0 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -88,7 +88,7 @@ async def process_json(app, body: dict) -> dict:
                 response['error'] = make_error('INVALID_PARAMS', err)
             else:
                 response['error'] = make_error('INTERNAL', err)
-                await app['webhooks'].spawn(report_error(app, err))
+                await app['webhooks'].spawn(report_error(app, err, body))
 
         finally:
             end = time.time()

From e9a8a3935c411d71a71b267d2745b7ada4af37f2 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 19 Jan 2020 23:14:01 -0500
Subject: [PATCH 04/46] Add gitignore

---
 _trial_temp/.gitignore | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 _trial_temp/.gitignore

diff --git a/_trial_temp/.gitignore b/_trial_temp/.gitignore
new file mode 100644
index 0000000..c7e09d7
--- /dev/null
+++ b/_trial_temp/.gitignore
@@ -0,0 +1 @@
+config/conf.json

From 6329ef1011b0ad9c71ae2e581de3a9bb50d8255e Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 19 Jan 2020 23:16:07 -0500
Subject: [PATCH 05/46] add .gitignore

---
 _trial_temp/.gitignore => .gitignore | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _trial_temp/.gitignore => .gitignore (100%)

diff --git a/_trial_temp/.gitignore b/.gitignore
similarity index 100%
rename from _trial_temp/.gitignore
rename to .gitignore

From ac69cd6966b59921e15599bc0ae212e403ff24d5 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 18 Feb 2020 14:36:38 -0500
Subject: [PATCH 06/46] Requires credential input for comment creation

---
 src/database/writes.py   |  5 ++---
 src/server/validation.py | 28 +++++++++++++++-------------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/database/writes.py b/src/database/writes.py
index 0e7b17d..b9d4b98 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -18,8 +18,7 @@ logger = logging.getLogger(__name__)
 
 def create_comment_or_error(conn, comment, claim_id=None, channel_id=None, channel_name=None,
                             signature=None, signing_ts=None, parent_id=None) -> dict:
-    if channel_id and channel_name:
-        insert_channel_or_error(conn, channel_name, channel_id)
+    insert_channel_or_error(conn, channel_name, channel_id)
     fn = db.insert_comment if parent_id is None else db.insert_reply
     comment_id = fn(
         conn=conn,
@@ -65,7 +64,7 @@ async def _abandon_comment(app, comment_id):  # DELETE
 
 
 async def create_comment(app, params):
-    if is_valid_base_comment(**params) and is_valid_credential_input(**params):
+    if is_valid_base_comment(**params):
         job = await app['comment_scheduler'].spawn(_create_comment(app, params))
         comment = await job.wait()
         if comment:
diff --git a/src/server/validation.py b/src/server/validation.py
index ad99f0c..43f05f6 100644
--- a/src/server/validation.py
+++ b/src/server/validation.py
@@ -51,23 +51,25 @@ def claim_id_is_valid(claim_id: str) -> bool:
 
 
 def is_valid_base_comment(comment: str, claim_id: str, parent_id: str = None, **kwargs) -> bool:
-    return comment is not None and body_is_valid(comment) and \
-           ((claim_id is not None and claim_id_is_valid(claim_id)) or
-            (parent_id is not None and comment_id_is_valid(parent_id)))
+    return comment and body_is_valid(comment) and \
+           ((claim_id and claim_id_is_valid(claim_id)) or  # parentid is used in place of claimid in replies
+            (parent_id and comment_id_is_valid(parent_id))) \
+           and is_valid_credential_input(**kwargs)
 
 
 def is_valid_credential_input(channel_id: str = None, channel_name: str = None,
                               signature: str = None, signing_ts: str = None, **kwargs) -> bool:
-    if channel_id or channel_name or signature or signing_ts:
-        try:
-            assert channel_id and channel_name and signature and signing_ts
-            assert is_valid_channel(channel_id, channel_name)
-            assert len(signature) == 128
-            assert signing_ts.isalnum()
-
-        except Exception:
-            return False
-    return True
+    try:
+        assert channel_id and channel_name and signature and signing_ts
+        assert is_valid_channel(channel_id, channel_name)
+        assert len(signature) == 128
+        assert signing_ts.isalnum()
+    except Exception as e:
+        logger.exception(f'Failed to validate channel: lbry://{channel_name}#{channel_id}, '
+                         f'signature: {signature} signing_ts: {signing_ts}')
+        return False
+    finally:
+        return True
 
 
 def validate_signature_from_claim(claim: dict, signature: typing.Union[str, bytes],

From 723026f967cab6b24f2431f9183a17e0030100af Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 18 Feb 2020 15:26:04 -0500
Subject: [PATCH 07/46] Proper kwarg management

---
 src/server/validation.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/server/validation.py b/src/server/validation.py
index 43f05f6..ef1c841 100644
--- a/src/server/validation.py
+++ b/src/server/validation.py
@@ -50,7 +50,8 @@ def claim_id_is_valid(claim_id: str) -> bool:
     return re.fullmatch('([a-z0-9]{40}|[A-Z0-9]{40})', claim_id) is not None
 
 
-def is_valid_base_comment(comment: str, claim_id: str, parent_id: str = None, **kwargs) -> bool:
+# default to None so params can be treated as kwargs; param count becomes more manageable
+def is_valid_base_comment(comment: str = None, claim_id: str = None, parent_id: str = None, **kwargs) -> bool:
     return comment and body_is_valid(comment) and \
            ((claim_id and claim_id_is_valid(claim_id)) or  # parentid is used in place of claimid in replies
             (parent_id and comment_id_is_valid(parent_id))) \
@@ -58,18 +59,19 @@ def is_valid_base_comment(comment: str, claim_id: str, parent_id: str = None, **
 
 
 def is_valid_credential_input(channel_id: str = None, channel_name: str = None,
-                              signature: str = None, signing_ts: str = None, **kwargs) -> bool:
+                              signature: str = None, signing_ts: str = None) -> bool:
     try:
-        assert channel_id and channel_name and signature and signing_ts
+        assert None not in (channel_id, channel_name, signature, signing_ts)
         assert is_valid_channel(channel_id, channel_name)
         assert len(signature) == 128
         assert signing_ts.isalnum()
+
+        return True
+
     except Exception as e:
         logger.exception(f'Failed to validate channel: lbry://{channel_name}#{channel_id}, '
                          f'signature: {signature} signing_ts: {signing_ts}')
         return False
-    finally:
-        return True
 
 
 def validate_signature_from_claim(claim: dict, signature: typing.Union[str, bytes],

From 220ceefbd242c63d67dee916714ca567c2c790f1 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 18 Feb 2020 15:27:29 -0500
Subject: [PATCH 08/46] Uses is_valid_base_comment instead of static proxy
 method

---
 test/test_server.py | 63 ++++++++++++++++++++++++++-------------------
 1 file changed, 37 insertions(+), 26 deletions(-)

diff --git a/test/test_server.py b/test/test_server.py
index 3c659d9..5cd9ea6 100644
--- a/test/test_server.py
+++ b/test/test_server.py
@@ -11,7 +11,6 @@ from faker.providers import misc
 
 from src.settings import config
 from src.server import app
-from src.server.validation import is_valid_channel
 from src.server.validation import is_valid_base_comment
 
 from test.testcase import AsyncioTestCase
@@ -97,22 +96,6 @@ class ServerTest(AsyncioTestCase):
     async def post_comment(self, **params):
         return await jsonrpc_post(self.url, 'create_comment', **params)
 
-    @staticmethod
-    def is_valid_message(comment=None, claim_id=None, parent_id=None,
-                         channel_name=None, channel_id=None, signature=None, signing_ts=None):
-        try:
-            assert is_valid_base_comment(comment, claim_id, parent_id)
-
-            if channel_name or channel_id or signature or signing_ts:
-                assert channel_id and channel_name and signature and signing_ts
-                assert is_valid_channel(channel_id, channel_name)
-                assert len(signature) == 128
-                assert signing_ts.isalnum()
-
-        except Exception:
-            return False
-        return True
-
     async def test01CreateCommentNoReply(self):
         anonymous_test = create_test_comments(
             ('claim_id', 'channel_id', 'channel_name', 'comment'),
@@ -122,13 +105,13 @@ class ServerTest(AsyncioTestCase):
             claim_id=None
         )
         for test in anonymous_test:
-            with self.subTest(test=test):
+            with self.subTest(test='null fields: ' + ', '.join(k for k, v in test.items() if not v)):
                 message = await self.post_comment(**test)
                 self.assertTrue('result' in message or 'error' in message)
                 if 'error' in message:
-                    self.assertFalse(self.is_valid_message(**test))
+                    self.assertFalse(is_valid_base_comment(**test))
                 else:
-                    self.assertTrue(self.is_valid_message(**test))
+                    self.assertTrue(is_valid_base_comment(**test))
 
     async def test02CreateNamedCommentsNoReply(self):
         named_test = create_test_comments(
@@ -144,9 +127,9 @@ class ServerTest(AsyncioTestCase):
                 message = await self.post_comment(**test)
                 self.assertTrue('result' in message or 'error' in message)
                 if 'error' in message:
-                    self.assertFalse(self.is_valid_message(**test))
+                    self.assertFalse(is_valid_base_comment(**test))
                 else:
-                    self.assertTrue(self.is_valid_message(**test))
+                    self.assertTrue(is_valid_base_comment(**test))
 
     async def test03CreateAllTestComments(self):
         test_all = create_test_comments(replace.keys(), **{
@@ -157,9 +140,9 @@ class ServerTest(AsyncioTestCase):
                 message = await self.post_comment(**test)
                 self.assertTrue('result' in message or 'error' in message)
                 if 'error' in message:
-                    self.assertFalse(self.is_valid_message(**test))
+                    self.assertFalse(is_valid_base_comment(**test))
                 else:
-                    self.assertTrue(self.is_valid_message(**test))
+                    self.assertTrue(is_valid_base_comment(**test))
 
     async def test04CreateAllReplies(self):
         claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8'
@@ -189,9 +172,37 @@ class ServerTest(AsyncioTestCase):
                     message = await self.post_comment(**test)
                     self.assertTrue('result' in message or 'error' in message)
                     if 'error' in message:
-                        self.assertFalse(self.is_valid_message(**test))
+                        self.assertFalse(is_valid_base_comment(**test))
                     else:
-                        self.assertTrue(self.is_valid_message(**test))
+                        self.assertTrue(is_valid_base_comment(**test))
+
+    async def testSlackWebhook(self):
+        claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8'
+        channel_name = '@name'
+        channel_id = fake.sha1()
+        signature = '{}'*64
+        signing_ts = '1234'
+
+        base = await self.post_comment(
+            channel_name=channel_name,
+            channel_id=channel_id,
+            comment='duplicate',
+            claim_id=claim_id,
+            signing_ts=signing_ts,
+            signature=signature
+        )
+
+        comment_id = base['result']['comment_id']
+
+        with self.subTest(test=comment_id):
+            await self.post_comment(
+                channel_name=channel_name,
+                channel_id=channel_id,
+                comment='duplicate',
+                claim_id=claim_id,
+                signing_ts=signing_ts,
+                signature=signature
+            )
 
 
 class ListCommentsTest(AsyncioTestCase):

From a825c6a4b9cadc45f150d0bd1f7e8c8e750a62cf Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 18 Feb 2020 15:28:46 -0500
Subject: [PATCH 09/46] Anonymous comment unit-test now tests against anonymous
 comments

---
 test/test_database.py | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/test/test_database.py b/test/test_database.py
index 347494d..bd10d16 100644
--- a/test/test_database.py
+++ b/test/test_database.py
@@ -1,4 +1,4 @@
-import unittest
+import sqlite3
 
 from random import randint
 import faker
@@ -53,21 +53,13 @@ class TestDatabaseOperations(DatabaseTestCase):
         self.assertEqual(reply['parent_id'], comment['comment_id'])
 
     def test02AnonymousComments(self):
-        comment = create_comment_or_error(
+        self.assertRaises(
+            sqlite3.IntegrityError,
+            create_comment_or_error,
             conn=self.conn,
             claim_id=self.claimId,
             comment='This is an ANONYMOUS comment'
         )
-        self.assertIsNotNone(comment)
-        previous_id = comment['comment_id']
-        reply = create_comment_or_error(
-            conn=self.conn,
-            claim_id=self.claimId,
-            comment='This is an unnamed response',
-            parent_id=previous_id
-        )
-        self.assertIsNotNone(reply)
-        self.assertEqual(reply['parent_id'], comment['comment_id'])
 
     def test03SignedComments(self):
         comment = create_comment_or_error(

From 77d499a0a3c30722f9b69fe63438a08b382b285e Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 18 Feb 2020 15:28:59 -0500
Subject: [PATCH 10/46] Removes unused tests

---
 test/test_database.py | 58 ++-----------------------------------------
 1 file changed, 2 insertions(+), 56 deletions(-)

diff --git a/test/test_database.py b/test/test_database.py
index bd10d16..c43bb0a 100644
--- a/test/test_database.py
+++ b/test/test_database.py
@@ -134,61 +134,7 @@ class TestDatabaseOperations(DatabaseTestCase):
             comment='this username is too short'
         )
 
-    def test05InsertRandomComments(self):
-        # TODO: Fix this test into something practical
-        self.skipTest('This is a bad test')
-        top_comments, claim_ids = generate_top_comments_random()
-        total = 0
-        success = 0
-        for _, comments in top_comments.items():
-            for i, comment in enumerate(comments):
-                with self.subTest(comment=comment):
-                    result = create_comment_or_error(self.conn, **comment)
-                    if result:
-                        success += 1
-                    comments[i] = result
-                    del comment
-                total += len(comments)
-        self.assertLessEqual(success, total)
-        self.assertGreater(success, 0)
-        success = 0
-        for reply in generate_replies_random(top_comments):
-            reply_id = create_comment_or_error(self.conn, **reply)
-            if reply_id:
-                success += 1
-        self.assertGreater(success, 0)
-        self.assertLess(success, total)
-        del top_comments
-        del claim_ids
-
-    def test06GenerateAndListComments(self):
-        # TODO: Make this test not suck
-        self.skipTest('this is a stupid test')
-        top_comments, claim_ids = generate_top_comments()
-        total, success = 0, 0
-        for _, comments in top_comments.items():
-            for i, comment in enumerate(comments):
-                result = create_comment_or_error(self.conn, **comment)
-                if result:
-                    success += 1
-                comments[i] = result
-                del comment
-            total += len(comments)
-        self.assertEqual(total, success)
-        self.assertGreater(total, 0)
-        for reply in generate_replies(top_comments):
-            create_comment_or_error(self.conn, **reply)
-        for claim_id in claim_ids:
-            comments_ids = get_comment_ids(self.conn, claim_id)
-            with self.subTest(comments_ids=comments_ids):
-                self.assertIs(type(comments_ids), list)
-                self.assertGreaterEqual(len(comments_ids), 0)
-                self.assertLessEqual(len(comments_ids), 50)
-                replies = get_comments_by_id(self.conn, comments_ids)
-                self.assertLessEqual(len(replies), 50)
-                self.assertEqual(len(replies), len(comments_ids))
-
-    def test07HideComments(self):
+    def test05HideComments(self):
         comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
         comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
         self.assertFalse(comment['is_hidden'])
@@ -201,7 +147,7 @@ class TestDatabaseOperations(DatabaseTestCase):
         comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
         self.assertTrue(comment['is_hidden'])
 
-    def test08DeleteComments(self):
+    def test06DeleteComments(self):
         comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
         comments = get_claim_comments(self.conn, self.claimId)
         match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items']))

From 0529fa7d015f959186380ce4ad5ce19ea9b0a7f0 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 27 Mar 2020 01:24:34 -0400
Subject: [PATCH 11/46] Implements DB model and preliminary select queries

---
 src/database/ddl.py | 159 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 src/database/ddl.py

diff --git a/src/database/ddl.py b/src/database/ddl.py
new file mode 100644
index 0000000..6593cd0
--- /dev/null
+++ b/src/database/ddl.py
@@ -0,0 +1,159 @@
+import json
+import logging
+
+import math
+import timeit
+
+import typing
+
+from peewee import ModelSelect
+from playhouse.shortcuts import model_to_dict
+from peewee import *
+
+
+def get_database_connection():
+    # for now it's an sqlite database
+    db = SqliteDatabase()
+    return db
+
+
+database = get_database_connection()
+
+
+class BaseModel(Model):
+    class Meta:
+        database = database
+
+
+class Channel(BaseModel):
+    claim_id = TextField(column_name='ClaimId', primary_key=True)
+    name = TextField(column_name='Name')
+
+    class Meta:
+        table_name = 'CHANNEL'
+
+
+class Comment(BaseModel):
+    comment = TextField(column_name='Body')
+    channel = ForeignKeyField(
+        backref='comments',
+        column_name='ChannelId',
+        field='claim_id',
+        model=Channel,
+        null=True
+    )
+    comment_id = TextField(column_name='CommentId', primary_key=True)
+    is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT FALSE")])
+    claim_id = TextField(column_name='LbryClaimId')
+    parent = ForeignKeyField(
+        column_name='ParentId',
+        field='comment_id',
+        model='self',
+        null=True,
+        backref='replies'
+    )
+    signature = TextField(column_name='Signature', null=True, unique=True)
+    signing_ts = TextField(column_name='SigningTs', null=True)
+    timestamp = IntegerField(column_name='Timestamp')
+
+    class Meta:
+        table_name = 'COMMENT'
+        indexes = (
+            (('author', 'comment_id'), False),
+            (('claim_id', 'comment_id'), False),
+        )
+
+
+COMMENT_FIELDS = [
+    Comment.comment,
+    Comment.comment_id,
+    Comment.claim_id,
+    Comment.timestamp,
+    Comment.signature,
+    Comment.signing_ts,
+    Comment.is_hidden,
+    Comment.parent.alias('parent_id'),
+]
+
+CHANNEL_FIELDS = [
+    Channel.claim_id.alias('channel_id'),
+    Channel.name.alias('channel_name')
+]
+
+
+def get_comment_list(claim_id: str = None, parent_id: str = None,
+                     top_level: bool = False, exclude_mode: str = None,
+                     page: int = 1, page_size: int = 50, expressions=None) -> dict:
+    query = Comment.select(*COMMENT_FIELDS, *CHANNEL_FIELDS)
+    if claim_id:
+        query = query.where(Comment.claim_id == claim_id)
+        if top_level:
+            query = query.where(Comment.parent.is_null())
+
+    if parent_id:
+        query = query.where(Comment.ParentId == parent_id)
+
+    if exclude_mode:
+        show_hidden = exclude_mode.lower() == 'hidden'
+        query = query.where((Comment.is_hidden == show_hidden))
+    total = query.count()
+    query = (query
+             .join(Channel, JOIN.LEFT_OUTER)
+             .where(expressions)
+             .order_by(Comment.timestamp.desc())
+             .paginate(page, page_size))
+    items = [clean(item) for item in query.dicts()]
+    # has_hidden_comments is deprecated
+    data = {
+        'page': page,
+        'page_size': page_size,
+        'total_pages': math.ceil(total / page_size),
+        'total_items': total,
+        'items': items,
+        'has_hidden_comments': exclude_mode is not None and exclude_mode == 'hidden',
+    }
+    return data
+
+
+def clean(thing: dict) -> dict:
+    return {k: v for k, v in thing.items() if v is not None}
+
+
+def get_comment(comment_id: str) -> dict:
+    try:
+        comment: Comment = Comment.get_by_id(comment_id)
+    except DoesNotExist as e:
+        raise ValueError from e
+    else:
+        as_dict = model_to_dict(comment)
+        if comment.channel:
+            as_dict.update({
+                'channel_id': comment.channel_id,
+                'channel_name': comment.channel.name,
+                'signature': comment.signature,
+                'signing_ts': comment.signing_ts,
+                'channel_url': f'lbry://{comment.channel.name}#{comment.channel_id}'
+            })
+        if comment.parent:
+            as_dict.update({
+                'parent_id': comment.parent_id
+            })
+        return clean(as_dict)
+
+
+if __name__ == '__main__':
+    logger = logging.getLogger('peewee')
+    logger.addHandler(logging.StreamHandler())
+    logger.setLevel(logging.DEBUG)
+
+    comment_list = get_comment_list(
+        page_size=1,
+        expressions=(Comment.channel.is_null())
+    )
+
+    comment = comment_list['items'].pop()
+    print(json.dumps(comment, indent=4))
+    other_comment = get_comment(comment['comment_id'])
+
+    print(json.dumps(other_comment, indent=4))
+    print(comment == other_comment)

From a581425a64988da11377ae964fc4180fdb5adf0a Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 27 Mar 2020 01:26:13 -0400
Subject: [PATCH 12/46] Shifts from JSON configuration to yml based

---
 setup.py            |  3 +++
 src/definitions.py  |  7 +++++++
 src/main.py         | 42 +++++++++++++++++++++++++++++++++++-------
 src/server/app.py   |  9 +++------
 src/settings.py     | 17 -----------------
 test/test_server.py |  5 ++++-
 6 files changed, 52 insertions(+), 31 deletions(-)
 create mode 100644 src/definitions.py
 delete mode 100644 src/settings.py

diff --git a/setup.py b/setup.py
index 923e82a..25087e4 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,8 @@ setup(
     data_files=[('config', ['config/conf.json',])],
     include_package_data=True,
     install_requires=[
+        'mysql-connector-python',
+        'pyyaml',
         'Faker>=1.0.7',
         'asyncio>=3.4.3',
         'aiohttp==3.5.4',
@@ -24,5 +26,6 @@ setup(
         'PyNaCl>=1.3.0',
         'requests',
         'cython',
+        'peewee'
     ]
 )
diff --git a/src/definitions.py b/src/definitions.py
new file mode 100644
index 0000000..9972e13
--- /dev/null
+++ b/src/definitions.py
@@ -0,0 +1,7 @@
+import os
+
+SRC_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(SRC_DIR)
+CONFIG_FILE = os.path.join(ROOT_DIR, 'config', 'conf.json')
+LOGGING_DIR = os.path.join(ROOT_DIR, 'logs')
+DATABASE_DIR = os.path.join(ROOT_DIR, 'database')
diff --git a/src/main.py b/src/main.py
index b31bcea..c22a82b 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1,13 +1,20 @@
 import argparse
+import json
+import yaml
 import logging
 import logging.config
+import os
 import sys
 
 from src.server.app import run_app
-from src.settings import config
+from src.definitions import LOGGING_DIR, CONFIG_FILE, DATABASE_DIR
 
 
-def config_logging_from_settings(conf):
+def setup_logging_from_config(conf: dict):
+    # set the logging directory here from the settings file
+    if not os.path.exists(LOGGING_DIR):
+        os.mkdir(LOGGING_DIR)
+
     _config = {
         "version": 1,
         "disable_existing_loggers": False,
@@ -32,7 +39,7 @@ def config_logging_from_settings(conf):
                 "level": "DEBUG",
                 "formatter": "standard",
                 "class": "logging.handlers.RotatingFileHandler",
-                "filename": conf['path']['debug_log'],
+                "filename": os.path.join(LOGGING_DIR, 'debug.log'),
                 "maxBytes": 10485760,
                 "backupCount": 5
             },
@@ -40,7 +47,7 @@ def config_logging_from_settings(conf):
                 "level": "ERROR",
                 "formatter": "standard",
                 "class": "logging.handlers.RotatingFileHandler",
-                "filename": conf['path']['error_log'],
+                "filename": os.path.join(LOGGING_DIR, 'error.log'),
                 "maxBytes": 10485760,
                 "backupCount": 5
             },
@@ -48,7 +55,7 @@ def config_logging_from_settings(conf):
                 "level": "NOTSET",
                 "formatter": "aiohttp",
                 "class": "logging.handlers.RotatingFileHandler",
-                "filename": conf['path']['server_log'],
+                "filename": os.path.join(LOGGING_DIR, 'server.log'),
                 "maxBytes": 10485760,
                 "backupCount": 5
             }
@@ -70,15 +77,36 @@ def config_logging_from_settings(conf):
     logging.config.dictConfig(_config)
 
 
+def get_config(filepath):
+    with open(filepath, 'r') as cfile:
+        config = yaml.load(cfile, Loader=yaml.FullLoader)
+    return config
+
+
+def setup_db_from_config(config: dict):
+    if 'sqlite' in config['database']:
+        if not os.path.exists(DATABASE_DIR):
+            os.mkdir(DATABASE_DIR)
+
+        config['db_path'] = os.path.join(
+            DATABASE_DIR, config['database']['sqlite']
+        )
+
+
 def main(argv=None):
     argv = argv or sys.argv[1:]
     parser = argparse.ArgumentParser(description='LBRY Comment Server')
     parser.add_argument('--port', type=int)
+    parser.add_argument('--config', type=str)
     args = parser.parse_args(argv)
-    config_logging_from_settings(config)
+
+    config = get_config(CONFIG_FILE) if not args.config else args.config
+    setup_logging_from_config(config)
+    setup_db_from_config(config)
+
     if args.port:
         config['port'] = args.port
-    config_logging_from_settings(config)
+
     run_app(config)
 
 
diff --git a/src/server/app.py b/src/server/app.py
index a25787b..446c378 100644
--- a/src/server/app.py
+++ b/src/server/app.py
@@ -75,12 +75,9 @@ class CommentDaemon:
         self.config = app['config']
 
         # configure the db file
-        if db_file:
-            app['db_path'] = db_file
-            app['backup'] = backup
-        else:
-            app['db_path'] = config['path']['database']
-            app['backup'] = backup or (app['db_path'] + '.backup')
+        app['db_path'] = db_file or config.get('db_path')
+        if app['db_path']:
+            app['backup'] = backup or '.'.join((app['db_path'], 'backup'))
 
         # configure the order of tasks to run during app lifetime
         app.on_startup.append(setup_db_schema)
diff --git a/src/settings.py b/src/settings.py
deleted file mode 100644
index 2d720c2..0000000
--- a/src/settings.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# cython: language_level=3
-import json
-import pathlib
-
-root_dir = pathlib.Path(__file__).parent.parent
-config_path = root_dir / 'config' / 'conf.json'
-
-
-def get_config(filepath):
-    with open(filepath, 'r') as cfile:
-        conf = json.load(cfile)
-    for key, path in conf['path'].items():
-        conf['path'][key] = str(root_dir / path)
-    return conf
-
-
-config = get_config(config_path)
diff --git a/test/test_server.py b/test/test_server.py
index 5cd9ea6..55fcdba 100644
--- a/test/test_server.py
+++ b/test/test_server.py
@@ -9,13 +9,16 @@ from faker.providers import internet
 from faker.providers import lorem
 from faker.providers import misc
 
-from src.settings import config
+from src.main import get_config, CONFIG_FILE
 from src.server import app
 from src.server.validation import is_valid_base_comment
 
 from test.testcase import AsyncioTestCase
 
 
+config = get_config(CONFIG_FILE)
+
+
 if 'slack_webhook' in config:
     config.pop('slack_webhook')
 

From cc20088b06e90a92b4f529f17f1c6b7e3e8d1171 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 27 Mar 2020 05:07:56 -0400
Subject: [PATCH 13/46] Implements all search queries

---
 src/database/ddl.py     | 104 +++++++++++++++++++++++++++++-----------
 src/database/queries.py |   2 +
 src/server/handles.py   |  16 +++----
 3 files changed, 85 insertions(+), 37 deletions(-)

diff --git a/src/database/ddl.py b/src/database/ddl.py
index 6593cd0..51d2de8 100644
--- a/src/database/ddl.py
+++ b/src/database/ddl.py
@@ -64,27 +64,34 @@ class Comment(BaseModel):
         )
 
 
-COMMENT_FIELDS = [
-    Comment.comment,
-    Comment.comment_id,
-    Comment.claim_id,
-    Comment.timestamp,
-    Comment.signature,
-    Comment.signing_ts,
-    Comment.is_hidden,
-    Comment.parent.alias('parent_id'),
-]
-
-CHANNEL_FIELDS = [
-    Channel.claim_id.alias('channel_id'),
-    Channel.name.alias('channel_name')
-]
+FIELDS = {
+    'comment': Comment.comment,
+    'comment_id': Comment.comment_id,
+    'claim_id': Comment.claim_id,
+    'timestamp': Comment.timestamp,
+    'signature': Comment.signature,
+    'signing_ts': Comment.signing_ts,
+    'is_hidden': Comment.is_hidden,
+    'parent_id': Comment.parent.alias('parent_id'),
+    'channel_id': Channel.claim_id.alias('channel_id'),
+    'channel_name': Channel.name.alias('channel_name'),
+    'channel_url': ('lbry://' + Channel.name + '#' + Channel.claim_id).alias('channel_url')
+}
 
 
-def get_comment_list(claim_id: str = None, parent_id: str = None,
-                     top_level: bool = False, exclude_mode: str = None,
-                     page: int = 1, page_size: int = 50, expressions=None) -> dict:
-    query = Comment.select(*COMMENT_FIELDS, *CHANNEL_FIELDS)
+def comment_list(claim_id: str = None, parent_id: str = None,
+                 top_level: bool = False, exclude_mode: str = None,
+                 page: int = 1, page_size: int = 50, expressions=None,
+                 select_fields: list = None, exclude_fields: list = None) -> dict:
+    fields = FIELDS.keys()
+    if exclude_fields:
+        fields -= set(exclude_fields)
+    if select_fields:
+        fields &= set(select_fields)
+    attributes = [FIELDS[field] for field in fields]
+    query = Comment.select(*attributes)
+
+    # todo: allow this process to be more automated, so it can just be an expression
     if claim_id:
         query = query.where(Comment.claim_id == claim_id)
         if top_level:
@@ -96,10 +103,13 @@ def get_comment_list(claim_id: str = None, parent_id: str = None,
     if exclude_mode:
         show_hidden = exclude_mode.lower() == 'hidden'
         query = query.where((Comment.is_hidden == show_hidden))
+
+    if expressions:
+        query = query.where(expressions)
+
     total = query.count()
     query = (query
              .join(Channel, JOIN.LEFT_OUTER)
-             .where(expressions)
              .order_by(Comment.timestamp.desc())
              .paginate(page, page_size))
     items = [clean(item) for item in query.dicts()]
@@ -132,7 +142,7 @@ def get_comment(comment_id: str) -> dict:
                 'channel_name': comment.channel.name,
                 'signature': comment.signature,
                 'signing_ts': comment.signing_ts,
-                'channel_url': f'lbry://{comment.channel.name}#{comment.channel_id}'
+                'channel_url': comment.channel.channel_url
             })
         if comment.parent:
             as_dict.update({
@@ -141,19 +151,55 @@ def get_comment(comment_id: str) -> dict:
         return clean(as_dict)
 
 
+def get_comment_ids(claim_id: str = None, parent_id: str = None,
+                    page: int = 1, page_size: int = 50, flattened=False) -> dict:
+    results = comment_list(
+        claim_id, parent_id,
+        top_level=(parent_id is None),
+        page=page, page_size=page_size,
+        select_fields=['comment_id', 'parent_id']
+    )
+    if flattened:
+        results.update({
+            'items': [item['comment_id'] for item in results['items']],
+            'replies': [(item['comment_id'], item.get('parent_id')) for item in results['items']]
+        })
+    return results
+
+
+def get_comments_by_id(comment_ids: typing.Union[list, tuple]) -> dict:
+    expression = Comment.comment_id.in_(comment_ids)
+    return comment_list(expressions=expression, page_size=len(comment_ids))
+
+
+def get_channel_from_comment_id(comment_id: str) -> dict:
+    try:
+        comment = Comment.get_by_id(comment_id)
+    except DoesNotExist as e:
+        raise ValueError from e
+    else:
+        channel = comment.channel
+        if not channel:
+            raise ValueError('The provided comment does not belong to a channel.')
+        return {
+            'channel_name': channel.name,
+            'channel_id': channel.claim_id,
+            'channel_url': 'lbry://' + channel.name + '#' + channel.claim_id
+        }
+
+
 if __name__ == '__main__':
     logger = logging.getLogger('peewee')
     logger.addHandler(logging.StreamHandler())
     logger.setLevel(logging.DEBUG)
 
-    comment_list = get_comment_list(
-        page_size=1,
-        expressions=(Comment.channel.is_null())
+    comments = comment_list(
+        page_size=20,
+        expressions=((Comment.timestamp < 1583272089) &
+                     (Comment.claim_id ** '420%'))
     )
 
-    comment = comment_list['items'].pop()
-    print(json.dumps(comment, indent=4))
-    other_comment = get_comment(comment['comment_id'])
+    ids = get_comment_ids('4207d2378bf4340e68c9d88faf7ee24ea1a1f95a')
 
-    print(json.dumps(other_comment, indent=4))
-    print(comment == other_comment)
+    print(json.dumps(comments, indent=4))
+    print(json.dumps(ids, indent=4))
\ No newline at end of file
diff --git a/src/database/queries.py b/src/database/queries.py
index 03815aa..662c22f 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -35,6 +35,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
                        page: int = 1, page_size: int = 50, top_level=False):
     with conn:
         if top_level:
+            # doesn't include any
             results = [clean(dict(row)) for row in conn.execute(
                 SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?",
                 (claim_id, page_size, page_size * (page - 1))
@@ -44,6 +45,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
                 (claim_id,)
             )
         elif parent_id is None:
+            # include all, no specific parent comment
             results = [clean(dict(row)) for row in conn.execute(
                 SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ",
                 (claim_id, page_size, page_size * (page - 1))
diff --git a/src/server/handles.py b/src/server/handles.py
index 6f629a0..c2d5f3e 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -55,16 +55,16 @@ async def handle_edit_comment(app, params):
 
 METHODS = {
     'ping': ping,
-    'get_claim_comments': handle_get_claim_comments,
-    'get_claim_hidden_comments': handle_get_claim_hidden_comments,
+    'get_claim_comments': handle_get_claim_comments,    # this gets used
+    'get_claim_hidden_comments': handle_get_claim_hidden_comments,  # this gets used
     'get_comment_ids': handle_get_comment_ids,
-    'get_comments_by_id': handle_get_comments_by_id,
-    'get_channel_from_comment_id': handle_get_channel_from_comment_id,
-    'create_comment': create_comment,
+    'get_comments_by_id': handle_get_comments_by_id,    # this gets used
+    'get_channel_from_comment_id': handle_get_channel_from_comment_id,  # this gets used
+    'create_comment': create_comment,   # this gets used
     'delete_comment': handle_abandon_comment,
-    'abandon_comment': handle_abandon_comment,
-    'hide_comments': handle_hide_comments,
-    'edit_comment': handle_edit_comment
+    'abandon_comment': handle_abandon_comment,  # this gets used
+    'hide_comments': handle_hide_comments,  # this gets used
+    'edit_comment': handle_edit_comment     # this gets used
 }
 
 

From 644e5e84771fc521744f60b3190c378a3e22f7b9 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 27 Mar 2020 15:59:57 -0400
Subject: [PATCH 14/46] adds get_channel_from_id & simplifies to all call
 comment_list

---
 src/database/ddl.py | 41 +++++++++--------------------------------
 1 file changed, 9 insertions(+), 32 deletions(-)

diff --git a/src/database/ddl.py b/src/database/ddl.py
index 51d2de8..491d521 100644
--- a/src/database/ddl.py
+++ b/src/database/ddl.py
@@ -130,25 +130,9 @@ def clean(thing: dict) -> dict:
 
 
 def get_comment(comment_id: str) -> dict:
-    try:
-        comment: Comment = Comment.get_by_id(comment_id)
-    except DoesNotExist as e:
-        raise ValueError from e
-    else:
-        as_dict = model_to_dict(comment)
-        if comment.channel:
-            as_dict.update({
-                'channel_id': comment.channel_id,
-                'channel_name': comment.channel.name,
-                'signature': comment.signature,
-                'signing_ts': comment.signing_ts,
-                'channel_url': comment.channel.channel_url
-            })
-        if comment.parent:
-            as_dict.update({
-                'parent_id': comment.parent_id
-            })
-        return clean(as_dict)
+    return (comment_list(expressions=(Comment.comment_id == comment_id), page_size=1)
+            .get('items')
+            .pop())
 
 
 def get_comment_ids(claim_id: str = None, parent_id: str = None,
@@ -173,19 +157,12 @@ def get_comments_by_id(comment_ids: typing.Union[list, tuple]) -> dict:
 
 
 def get_channel_from_comment_id(comment_id: str) -> dict:
-    try:
-        comment = Comment.get_by_id(comment_id)
-    except DoesNotExist as e:
-        raise ValueError from e
-    else:
-        channel = comment.channel
-        if not channel:
-            raise ValueError('The provided comment does not belong to a channel.')
-        return {
-            'channel_name': channel.name,
-            'channel_id': channel.claim_id,
-            'channel_url': 'lbry://' + channel.name + '#' + channel.claim_id
-        }
+    results = comment_list(
+        expressions=(Comment.comment_id == comment_id),
+        select_fields=['channel_name', 'channel_id', 'channel_url'],
+        page_size=1
+    )
+    return results['items'].pop()
 
 
 if __name__ == '__main__':

From 63f2c7e9e06a6d08d4c33d155584c08b2486a2f4 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 27 Mar 2020 17:44:22 -0400
Subject: [PATCH 15/46] Moves the clean method to misc.py, moves misc.py

---
 src/database/ddl.py      | 15 ++++++---------
 src/database/writes.py   |  2 +-
 src/{server => }/misc.py |  4 ++++
 src/server/handles.py    |  2 +-
 4 files changed, 12 insertions(+), 11 deletions(-)
 rename src/{server => }/misc.py (86%)

diff --git a/src/database/ddl.py b/src/database/ddl.py
index 491d521..4a0d9c3 100644
--- a/src/database/ddl.py
+++ b/src/database/ddl.py
@@ -1,14 +1,15 @@
 import json
+import time
+
 import logging
-
 import math
-import timeit
-
 import typing
 
-from peewee import ModelSelect
-from playhouse.shortcuts import model_to_dict
 from peewee import *
+import nacl.hash
+
+from src.server.validation import is_valid_base_comment
+from src.misc import clean
 
 
 def get_database_connection():
@@ -125,10 +126,6 @@ def comment_list(claim_id: str = None, parent_id: str = None,
     return data
 
 
-def clean(thing: dict) -> dict:
-    return {k: v for k, v in thing.items() if v is not None}
-
-
 def get_comment(comment_id: str) -> dict:
     return (comment_list(expressions=(Comment.comment_id == comment_id), page_size=1)
             .get('items')
diff --git a/src/database/writes.py b/src/database/writes.py
index b9d4b98..9f13f3f 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -7,7 +7,7 @@ from src.server.validation import is_valid_base_comment
 from src.server.validation import is_valid_credential_input
 from src.server.validation import validate_signature_from_claim
 from src.server.validation import body_is_valid
-from src.server.misc import get_claim_from_id
+from src.misc import get_claim_from_id
 from src.server.external import send_notifications
 from src.server.external import send_notification
 import src.database.queries as db
diff --git a/src/server/misc.py b/src/misc.py
similarity index 86%
rename from src/server/misc.py
rename to src/misc.py
index 4f620e6..e9d5be8 100644
--- a/src/server/misc.py
+++ b/src/misc.py
@@ -20,3 +20,7 @@ def clean_input_params(kwargs: dict):
             kwargs[k] = v.strip()
             if k in ID_LIST:
                 kwargs[k] = v.lower()
+
+
+def clean(thing: dict) -> dict:
+    return {k: v for k, v in thing.items() if v is not None}
\ No newline at end of file
diff --git a/src/server/handles.py b/src/server/handles.py
index c2d5f3e..a63504b 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -9,7 +9,7 @@ import src.database.queries as db
 from src.database.writes import abandon_comment, create_comment
 from src.database.writes import hide_comments
 from src.database.writes import edit_comment
-from src.server.misc import clean_input_params
+from src.misc import clean_input_params
 from src.server.errors import make_error, report_error
 
 logger = logging.getLogger(__name__)

From 510f2a5d29ca2dd71a960fc69472e0228d7daf96 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 27 Mar 2020 17:44:51 -0400
Subject: [PATCH 16/46] Adds comment create logic

---
 src/database/ddl.py | 53 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/src/database/ddl.py b/src/database/ddl.py
index 4a0d9c3..7c3c145 100644
--- a/src/database/ddl.py
+++ b/src/database/ddl.py
@@ -159,9 +159,60 @@ def get_channel_from_comment_id(comment_id: str) -> dict:
         select_fields=['channel_name', 'channel_id', 'channel_url'],
         page_size=1
     )
+    # todo: make the return type here consistent
     return results['items'].pop()
 
 
+def create_comment_id(comment: str, channel_id: str, timestamp: int):
+    # We convert the timestamp from seconds into minutes
+    # to prevent spammers from commenting the same BS everywhere.
+    nearest_minute = str(math.floor(timestamp))
+
+    # don't use claim_id for the comment_id anymore so comments
+    # are not unique to just one claim
+    prehash = b':'.join([
+        comment.encode(),
+        channel_id.encode(),
+        nearest_minute.encode()
+    ])
+    return nacl.hash.sha256(prehash).decode()
+
+
+def create_comment(comment: str = None, claim_id: str = None,
+                   parent_id: str = None, channel_id: str = None,
+                   channel_name: str = None, signature: str = None,
+                   signing_ts: str = None) -> dict:
+    if not is_valid_base_comment(
+            comment=comment,
+            claim_id=claim_id,
+            parent_id=parent_id,
+            channel_id=channel_id,
+            channel_name=channel_name,
+            signature=signature,
+            signing_ts=signing_ts
+    ):
+        raise ValueError('Invalid Parameters given for comment')
+
+    channel, _ = Channel.get_or_create(name=channel_name, claim_id=channel_id)
+    if parent_id:
+        parent: Comment = Comment.get_by_id(parent_id)
+        claim_id = parent.claim_id
+
+    timestamp = int(time.time())
+    comment_id = create_comment_id(comment, channel_id, timestamp)
+    new_comment = Comment.create(
+        claim_id=claim_id,
+        comment_id=comment_id,
+        comment=comment,
+        parent=parent_id,
+        channel=channel,
+        signature=signature,
+        signing_ts=signing_ts,
+        timestamp=timestamp
+    )
+    return get_comment(new_comment.comment_id)
+
+
 if __name__ == '__main__':
     logger = logging.getLogger('peewee')
     logger.addHandler(logging.StreamHandler())
@@ -176,4 +227,4 @@ if __name__ == '__main__':
     ids = get_comment_ids('4207d2378bf4340e68c9d88faf7ee24ea1a1f95a')
 
     print(json.dumps(comments, indent=4))
-    print(json.dumps(ids, indent=4))
\ No newline at end of file
+    print(json.dumps(ids, indent=4))

From 8138e7166861c7cff5e2786fdda381d82b1e50b1 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 30 Mar 2020 18:02:20 -0400
Subject: [PATCH 17/46] Sets database dynamically

---
 src/database/ddl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/database/ddl.py b/src/database/ddl.py
index 7c3c145..5778965 100644
--- a/src/database/ddl.py
+++ b/src/database/ddl.py
@@ -18,12 +18,15 @@ def get_database_connection():
     return db
 
 
+
+database_proxy = DatabaseProxy()
 database = get_database_connection()
+database_proxy.initialize(database)
 
 
 class BaseModel(Model):
     class Meta:
-        database = database
+        database = database_proxy
 
 
 class Channel(BaseModel):

From a22b4a9162454fec9fa3578eb5d0d6badf0ac0f3 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 30 Mar 2020 18:04:20 -0400
Subject: [PATCH 18/46] Adds delete, edit, and hide operations

---
 src/database/ddl.py | 58 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 11 deletions(-)

diff --git a/src/database/ddl.py b/src/database/ddl.py
index 5778965..0983b74 100644
--- a/src/database/ddl.py
+++ b/src/database/ddl.py
@@ -203,17 +203,53 @@ def create_comment(comment: str = None, claim_id: str = None,
 
     timestamp = int(time.time())
     comment_id = create_comment_id(comment, channel_id, timestamp)
-    new_comment = Comment.create(
-        claim_id=claim_id,
-        comment_id=comment_id,
-        comment=comment,
-        parent=parent_id,
-        channel=channel,
-        signature=signature,
-        signing_ts=signing_ts,
-        timestamp=timestamp
-    )
-    return get_comment(new_comment.comment_id)
+    with database_proxy.atomic():
+        new_comment = Comment.create(
+            claim_id=claim_id,
+            comment_id=comment_id,
+            comment=comment,
+            parent=parent_id,
+            channel=channel,
+            signature=signature,
+            signing_ts=signing_ts,
+            timestamp=timestamp
+        )
+        return get_comment(new_comment.comment_id)
+
+
+def delete_comment(comment_id: str) -> bool:
+    try:
+        comment: Comment = Comment.get_by_id(comment_id)
+    except DoesNotExist as e:
+        raise ValueError from e
+    else:
+        with database_proxy.atomic():
+            return 0 < comment.delete_instance(True, delete_nullable=True)
+
+
+def edit_comment(comment_id: str, new_comment: str, new_sig: str, new_ts: str) -> bool:
+    try:
+        comment: Comment = Comment.get_by_id(comment_id)
+    except DoesNotExist as e:
+        raise ValueError from e
+    else:
+        with database_proxy.atomic():
+            comment.comment = new_comment
+            comment.signature = new_sig
+            comment.signing_ts = new_ts
+
+            # todo: add a 'last-modified' timestamp
+            comment.timestamp = int(time.time())
+            return comment.save() > 0
+
+
+def set_hidden_flag(comment_ids: typing.List[str], hidden=True) -> bool:
+    # sets `is_hidden` flag for all `comment_ids` to the `hidden` param
+    with database_proxy.atomic():
+        update = (Comment
+                  .update(is_hidden=hidden)
+                  .where(Comment.comment_id.in_(comment_ids)))
+        return update.execute() > 0
 
 
 if __name__ == '__main__':

From aee12eba54bc11c0f8323782be428e9662c3d820 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 30 Mar 2020 21:58:07 -0400
Subject: [PATCH 19/46] removes inherited base database in favor of binding api

---
 src/database/{ddl.py => models.py} | 59 ++++++++++++------------------
 1 file changed, 23 insertions(+), 36 deletions(-)
 rename src/database/{ddl.py => models.py} (86%)

diff --git a/src/database/ddl.py b/src/database/models.py
similarity index 86%
rename from src/database/ddl.py
rename to src/database/models.py
index 0983b74..0658c33 100644
--- a/src/database/ddl.py
+++ b/src/database/models.py
@@ -12,24 +12,15 @@ from src.server.validation import is_valid_base_comment
 from src.misc import clean
 
 
-def get_database_connection():
-    # for now it's an sqlite database
-    db = SqliteDatabase()
-    return db
+def get_database_connection(dbms, db_name, **params):
+    if dbms == 'mysql':
+        return MySQLDatabase(db_name, **params)
+    else:
+        # return SqliteDatabase('/home/oleg/PycharmProjects/comment-server/database/default_pw.db')
+        return SqliteDatabase(db_name)
 
 
-
-database_proxy = DatabaseProxy()
-database = get_database_connection()
-database_proxy.initialize(database)
-
-
-class BaseModel(Model):
-    class Meta:
-        database = database_proxy
-
-
-class Channel(BaseModel):
+class Channel(Model):
     claim_id = TextField(column_name='ClaimId', primary_key=True)
     name = TextField(column_name='Name')
 
@@ -37,7 +28,7 @@ class Channel(BaseModel):
         table_name = 'CHANNEL'
 
 
-class Comment(BaseModel):
+class Comment(Model):
     comment = TextField(column_name='Body')
     channel = ForeignKeyField(
         backref='comments',
@@ -47,7 +38,7 @@ class Comment(BaseModel):
         null=True
     )
     comment_id = TextField(column_name='CommentId', primary_key=True)
-    is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT FALSE")])
+    is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")])
     claim_id = TextField(column_name='LbryClaimId')
     parent = ForeignKeyField(
         column_name='ParentId',
@@ -63,7 +54,7 @@ class Comment(BaseModel):
     class Meta:
         table_name = 'COMMENT'
         indexes = (
-            (('author', 'comment_id'), False),
+            (('channel', 'comment_id'), False),
             (('claim_id', 'comment_id'), False),
         )
 
@@ -203,8 +194,7 @@ def create_comment(comment: str = None, claim_id: str = None,
 
     timestamp = int(time.time())
     comment_id = create_comment_id(comment, channel_id, timestamp)
-    with database_proxy.atomic():
-        new_comment = Comment.create(
+    new_comment = Comment.create(
             claim_id=claim_id,
             comment_id=comment_id,
             comment=comment,
@@ -214,7 +204,7 @@ def create_comment(comment: str = None, claim_id: str = None,
             signing_ts=signing_ts,
             timestamp=timestamp
         )
-        return get_comment(new_comment.comment_id)
+    return get_comment(new_comment.comment_id)
 
 
 def delete_comment(comment_id: str) -> bool:
@@ -223,8 +213,7 @@ def delete_comment(comment_id: str) -> bool:
     except DoesNotExist as e:
         raise ValueError from e
     else:
-        with database_proxy.atomic():
-            return 0 < comment.delete_instance(True, delete_nullable=True)
+        return 0 < comment.delete_instance(True, delete_nullable=True)
 
 
 def edit_comment(comment_id: str, new_comment: str, new_sig: str, new_ts: str) -> bool:
@@ -233,23 +222,21 @@ def edit_comment(comment_id: str, new_comment: str, new_sig: str, new_ts: str) -
     except DoesNotExist as e:
         raise ValueError from e
     else:
-        with database_proxy.atomic():
-            comment.comment = new_comment
-            comment.signature = new_sig
-            comment.signing_ts = new_ts
+        comment.comment = new_comment
+        comment.signature = new_sig
+        comment.signing_ts = new_ts
 
-            # todo: add a 'last-modified' timestamp
-            comment.timestamp = int(time.time())
-            return comment.save() > 0
+        # todo: add a 'last-modified' timestamp
+        comment.timestamp = int(time.time())
+        return comment.save() > 0
 
 
 def set_hidden_flag(comment_ids: typing.List[str], hidden=True) -> bool:
     # sets `is_hidden` flag for all `comment_ids` to the `hidden` param
-    with database_proxy.atomic():
-        update = (Comment
-                  .update(is_hidden=hidden)
-                  .where(Comment.comment_id.in_(comment_ids)))
-        return update.execute() > 0
+    update = (Comment
+              .update(is_hidden=hidden)
+              .where(Comment.comment_id.in_(comment_ids)))
+    return update.execute() > 0
 
 
 if __name__ == '__main__':

From 45733d2dc4962714508ff45e3af3dc027f35ed49 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 30 Mar 2020 21:58:37 -0400
Subject: [PATCH 20/46] Remove queries.py

---
 src/database/queries.py | 292 ----------------------------------------
 1 file changed, 292 deletions(-)
 delete mode 100644 src/database/queries.py

diff --git a/src/database/queries.py b/src/database/queries.py
deleted file mode 100644
index 662c22f..0000000
--- a/src/database/queries.py
+++ /dev/null
@@ -1,292 +0,0 @@
-import atexit
-import logging
-import math
-import sqlite3
-import time
-import typing
-
-import nacl.hash
-
-from src.database.schema import CREATE_TABLES_QUERY
-
-logger = logging.getLogger(__name__)
-
-SELECT_COMMENTS_ON_CLAIMS = """
-    SELECT comment, comment_id, claim_id, timestamp, is_hidden, parent_id,
-        channel_name, channel_id, channel_url, signature, signing_ts 
-    FROM COMMENTS_ON_CLAIMS 
-"""
-
-
-def clean(thing: dict) -> dict:
-    if 'is_hidden' in thing:
-        thing.update({'is_hidden': bool(thing['is_hidden'])})
-    return {k: v for k, v in thing.items() if v is not None}
-
-
-def obtain_connection(filepath: str = None, row_factory: bool = True):
-    connection = sqlite3.connect(filepath)
-    if row_factory:
-        connection.row_factory = sqlite3.Row
-    return connection
-
-
-def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = None,
-                       page: int = 1, page_size: int = 50, top_level=False):
-    with conn:
-        if top_level:
-            # doesn't include any
-            results = [clean(dict(row)) for row in conn.execute(
-                SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?",
-                (claim_id, page_size, page_size * (page - 1))
-            )]
-            count = conn.execute(
-                "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id IS NULL",
-                (claim_id,)
-            )
-        elif parent_id is None:
-            # include all, no specific parent comment
-            results = [clean(dict(row)) for row in conn.execute(
-                SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ",
-                (claim_id, page_size, page_size * (page - 1))
-            )]
-            count = conn.execute(
-                "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ?",
-                (claim_id,)
-            )
-        else:
-            results = [clean(dict(row)) for row in conn.execute(
-                SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? ",
-                (claim_id, parent_id, page_size, page_size * (page - 1))
-            )]
-            count = conn.execute(
-                "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id = ?",
-                (claim_id, parent_id)
-            )
-        count = tuple(count.fetchone())[0]
-        return {
-            'items': results,
-            'page': page,
-            'page_size': page_size,
-            'total_pages': math.ceil(count / page_size),
-            'total_items': count,
-            'has_hidden_comments': claim_has_hidden_comments(conn, claim_id)
-        }
-
-
-def get_claim_hidden_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50):
-    with conn:
-        results = conn.execute(
-            SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden IS ? LIMIT ? OFFSET ?",
-            (claim_id, hidden, page_size, page_size * (page - 1))
-        )
-        count = conn.execute(
-            "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden IS ?", (claim_id, hidden)
-        )
-    results = [clean(dict(row)) for row in results.fetchall()]
-    count = tuple(count.fetchone())[0]
-
-    return {
-        'items': results,
-        'page': page,
-        'page_size': page_size,
-        'total_pages': math.ceil(count / page_size),
-        'total_items': count,
-        'has_hidden_comments': claim_has_hidden_comments(conn, claim_id)
-    }
-
-
-def claim_has_hidden_comments(conn, claim_id):
-    with conn:
-        result = conn.execute(
-            "SELECT COUNT(DISTINCT is_hidden) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden IS 1",
-            (claim_id,)
-        )
-        return bool(tuple(result.fetchone())[0])
-
-
-def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str,
-                   channel_id: str = None, signature: str = None, signing_ts: str = None, **extra) -> str:
-    timestamp = int(time.time())
-    prehash = b':'.join((claim_id.encode(), comment.encode(), str(timestamp).encode(),))
-    comment_id = nacl.hash.sha256(prehash).decode()
-    with conn:
-        curs = conn.execute(
-            """
-            INSERT INTO COMMENT(CommentId, LbryClaimId, ChannelId, Body, ParentId, 
-                                    Timestamp, Signature, SigningTs, IsHidden) 
-            VALUES (:comment_id, :claim_id, :channel_id, :comment, NULL,
-                    :timestamp, :signature, :signing_ts, 0) """,
-            {
-                'comment_id': comment_id,
-                'claim_id': claim_id,
-                'channel_id': channel_id,
-                'comment': comment,
-                'timestamp': timestamp,
-                'signature': signature,
-                'signing_ts': signing_ts
-            }
-        )
-        logging.info('attempted to insert comment with comment_id [%s] | %d rows affected', comment_id, curs.rowcount)
-    return comment_id
-
-
-def insert_reply(conn: sqlite3.Connection, comment: str, parent_id: str,
-                 channel_id: str = None, signature: str = None,
-                 signing_ts: str = None, **extra) -> str:
-    timestamp = int(time.time())
-    prehash = b':'.join((parent_id.encode(), comment.encode(), str(timestamp).encode(),))
-    comment_id = nacl.hash.sha256(prehash).decode()
-    with conn:
-        curs = conn.execute(
-            """
-            INSERT INTO COMMENT 
-                (CommentId, LbryClaimId, ChannelId, Body, ParentId, Signature, Timestamp, SigningTs, IsHidden) 
-            SELECT :comment_id, LbryClaimId, :channel_id, :comment, :parent_id, :signature, :timestamp, :signing_ts, 0
-            FROM COMMENT WHERE CommentId = :parent_id
-            """, {
-                'comment_id': comment_id,
-                'parent_id': parent_id,
-                'timestamp': timestamp,
-                'comment': comment,
-                'channel_id': channel_id,
-                'signature': signature,
-                'signing_ts': signing_ts
-            }
-        )
-        logging.info('attempted to insert reply with comment_id [%s] | %d rows affected', comment_id, curs.rowcount)
-    return comment_id
-
-
-def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict:
-    with conn:
-        curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,))
-        thing = curry.fetchone()
-        return clean(dict(thing)) if thing else None
-
-
-def get_comment_ids(conn: sqlite3.Connection, claim_id: str, parent_id: str = None, page=1, page_size=50):
-    """ Just return a list of the comment IDs that are associated with the given claim_id.
-    If get_all is specified then it returns all the IDs, otherwise only the IDs at that level.
-    if parent_id is left null then it only returns the top level comments.
-
-    For pagination the parameters are:
-        get_all XOR (page_size + page)
-    """
-    with conn:
-        if parent_id is None:
-            curs = conn.execute("""
-                    SELECT comment_id FROM COMMENTS_ON_CLAIMS
-                    WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?
-                """, (claim_id, page_size, page_size * abs(page - 1),)
-                                )
-        else:
-            curs = conn.execute("""
-                    SELECT comment_id FROM COMMENTS_ON_CLAIMS
-                    WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ?
-                """, (claim_id, parent_id, page_size, page_size * abs(page - 1),)
-                                )
-    return [tuple(row)[0] for row in curs.fetchall()]
-
-
-def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.Union[list, None]:
-    """ Returns a list containing the comment data associated with each ID within the list"""
-    # format the input, under the assumption that the
-    placeholders = ', '.join('?' for _ in comment_ids)
-    with conn:
-        return [clean(dict(row)) for row in conn.execute(
-            SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})',
-            tuple(comment_ids)
-        )]
-
-
-def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str) -> bool:
-    with conn:
-        curs = conn.execute("DELETE FROM COMMENT WHERE CommentId = ?", (comment_id,))
-        return bool(curs.rowcount)
-
-
-def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str):
-    with conn:
-        curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name)  VALUES (?, ?)', (channel_id, channel_name))
-        return bool(curs.rowcount)
-
-
-def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str):
-    with conn:
-        channel = conn.execute(
-            "SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?", (comment_id,)
-        ).fetchone()
-        return dict(channel) if channel else {}
-
-
-def get_claim_ids_from_comment_ids(conn: sqlite3.Connection, comment_ids: list):
-    with conn:
-        cids = conn.execute(
-            f""" SELECT  CommentId as comment_id, LbryClaimId AS claim_id FROM COMMENT 
-            WHERE CommentId IN ({', '.join('?' for _ in comment_ids)}) """,
-            tuple(comment_ids)
-        )
-        return {row['comment_id']: row['claim_id'] for row in cids.fetchall()}
-
-
-def hide_comments_by_id(conn: sqlite3.Connection, comment_ids: list) -> bool:
-    with conn:
-        curs = conn.cursor()
-        curs.executemany(
-            "UPDATE COMMENT SET IsHidden = 1 WHERE CommentId = ?",
-            [[c] for c in comment_ids]
-        )
-        return bool(curs.rowcount)
-
-
-def edit_comment_by_id(conn: sqlite3.Connection, comment_id: str, comment: str,
-                       signature: str, signing_ts: str) -> bool:
-    with conn:
-        curs = conn.execute(
-            """
-                UPDATE COMMENT 
-                SET Body = :comment, Signature = :signature, SigningTs = :signing_ts
-                WHERE CommentId = :comment_id
-            """,
-            {
-                'comment': comment,
-                'signature': signature,
-                'signing_ts': signing_ts,
-                'comment_id': comment_id
-            })
-        logger.info("updated comment with `comment_id`: %s", comment_id)
-        return bool(curs.rowcount)
-
-
-class DatabaseWriter(object):
-    _writer = None
-
-    def __init__(self, db_file):
-        if not DatabaseWriter._writer:
-            self.conn = obtain_connection(db_file)
-            DatabaseWriter._writer = self
-            atexit.register(self.cleanup)
-            logging.info('Database writer has been created at %s', repr(self))
-        else:
-            logging.warning('Someone attempted to insantiate DatabaseWriter')
-            raise TypeError('Database Writer already exists!')
-
-    def cleanup(self):
-        logging.info('Cleaning up database writer')
-        self.conn.close()
-        DatabaseWriter._writer = None
-
-    @property
-    def connection(self):
-        return self.conn
-
-
-def setup_database(db_path):
-    with sqlite3.connect(db_path) as conn:
-        conn.executescript(CREATE_TABLES_QUERY)
-
-
-def backup_database(conn: sqlite3.Connection, back_fp):
-    with sqlite3.connect(back_fp) as back:
-        conn.backup(back)

From dba14460cc8845e707a6e552d36d1bf4da9c118d Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 30 Mar 2020 21:59:26 -0400
Subject: [PATCH 21/46] Unittests using peewee binds instead of direct sqlite
 connection

---
 test/test_database.py | 222 +++++++++++++++++++++++++-----------------
 test/testcase.py      |  49 ++++++----
 2 files changed, 160 insertions(+), 111 deletions(-)

diff --git a/test/test_database.py b/test/test_database.py
index c43bb0a..bf25bba 100644
--- a/test/test_database.py
+++ b/test/test_database.py
@@ -1,18 +1,13 @@
-import sqlite3
-
 from random import randint
 import faker
 from faker.providers import internet
 from faker.providers import lorem
 from faker.providers import misc
 
-from src.database.queries import get_comments_by_id
-from src.database.queries import get_comment_ids
-from src.database.queries import get_claim_comments
-from src.database.queries import get_claim_hidden_comments
-from src.database.writes import create_comment_or_error
-from src.database.queries import hide_comments_by_id
-from src.database.queries import delete_comment_by_id
+from src.database.models import create_comment
+from src.database.models import delete_comment
+from src.database.models import comment_list, get_comment, get_comments_by_id
+from src.database.models import set_hidden_flag
 from test.testcase import DatabaseTestCase
 
 fake = faker.Faker()
@@ -27,26 +22,25 @@ class TestDatabaseOperations(DatabaseTestCase):
         self.claimId = '529357c3422c6046d3fec76be2358004ba22e340'
 
     def test01NamedComments(self):
-        comment = create_comment_or_error(
-            conn=self.conn,
+        comment = create_comment(
             claim_id=self.claimId,
             comment='This is a named comment',
             channel_name='@username',
             channel_id='529357c3422c6046d3fec76be2358004ba22abcd',
-            signature=fake.uuid4(),
+            signature='22'*64,
             signing_ts='aaa'
         )
         self.assertIsNotNone(comment)
         self.assertNotIn('parent_in', comment)
+
         previous_id = comment['comment_id']
-        reply = create_comment_or_error(
-            conn=self.conn,
+        reply = create_comment(
             claim_id=self.claimId,
             comment='This is a named response',
             channel_name='@another_username',
             channel_id='529357c3422c6046d3fec76be2358004ba224bcd',
             parent_id=previous_id,
-            signature=fake.uuid4(),
+            signature='11'*64,
             signing_ts='aaa'
         )
         self.assertIsNotNone(reply)
@@ -54,34 +48,32 @@ class TestDatabaseOperations(DatabaseTestCase):
 
     def test02AnonymousComments(self):
         self.assertRaises(
-            sqlite3.IntegrityError,
-            create_comment_or_error,
-            conn=self.conn,
+            ValueError,
+            create_comment,
             claim_id=self.claimId,
             comment='This is an ANONYMOUS comment'
         )
 
     def test03SignedComments(self):
-        comment = create_comment_or_error(
-            conn=self.conn,
+        comment = create_comment(
             claim_id=self.claimId,
             comment='I like big butts and i cannot lie',
             channel_name='@sirmixalot',
             channel_id='529357c3422c6046d3fec76be2358005ba22abcd',
-            signature=fake.uuid4(),
+            signature='24'*64,
             signing_ts='asdasd'
         )
         self.assertIsNotNone(comment)
         self.assertIn('signing_ts', comment)
+
         previous_id = comment['comment_id']
-        reply = create_comment_or_error(
-            conn=self.conn,
+        reply = create_comment(
             claim_id=self.claimId,
             comment='This is a LBRY verified response',
             channel_name='@LBRY',
             channel_id='529357c3422c6046d3fec76be2358001ba224bcd',
             parent_id=previous_id,
-            signature=fake.uuid4(),
+            signature='12'*64,
             signing_ts='sfdfdfds'
         )
         self.assertIsNotNone(reply)
@@ -90,75 +82,109 @@ class TestDatabaseOperations(DatabaseTestCase):
 
     def test04UsernameVariations(self):
         self.assertRaises(
-            AssertionError,
-            callable=create_comment_or_error,
-            conn=self.conn,
+            ValueError,
+            create_comment,
             claim_id=self.claimId,
             channel_name='$#(@#$@#$',
             channel_id='529357c3422c6046d3fec76be2358001ba224b23',
-            comment='this is an invalid username'
+            comment='this is an invalid username',
+            signature='1' * 128,
+            signing_ts='123'
         )
-        valid_username = create_comment_or_error(
-            conn=self.conn,
+
+        valid_username = create_comment(
             claim_id=self.claimId,
             channel_name='@' + 'a' * 255,
             channel_id='529357c3422c6046d3fec76be2358001ba224b23',
-            comment='this is a valid username'
+            comment='this is a valid username',
+            signature='1'*128,
+            signing_ts='123'
         )
         self.assertIsNotNone(valid_username)
-        self.assertRaises(AssertionError,
-                          callable=create_comment_or_error,
-                          conn=self.conn,
-                          claim_id=self.claimId,
-                          channel_name='@' + 'a' * 256,
-                          channel_id='529357c3422c6046d3fec76be2358001ba224b23',
-                          comment='this username is too long'
-                          )
 
         self.assertRaises(
-            AssertionError,
-            callable=create_comment_or_error,
-            conn=self.conn,
+            ValueError,
+            create_comment,
+            claim_id=self.claimId,
+            channel_name='@' + 'a' * 256,
+            channel_id='529357c3422c6046d3fec76be2358001ba224b23',
+            comment='this username is too long',
+            signature='2' * 128,
+            signing_ts='123'
+        )
+
+        self.assertRaises(
+            ValueError,
+            create_comment,
             claim_id=self.claimId,
             channel_name='',
             channel_id='529357c3422c6046d3fec76be2358001ba224b23',
-            comment='this username should not default to ANONYMOUS'
+            comment='this username should not default to ANONYMOUS',
+            signature='3' * 128,
+            signing_ts='123'
         )
+
         self.assertRaises(
-            AssertionError,
-            callable=create_comment_or_error,
-            conn=self.conn,
+            ValueError,
+            create_comment,
             claim_id=self.claimId,
             channel_name='@',
             channel_id='529357c3422c6046d3fec76be2358001ba224b23',
-            comment='this username is too short'
+            comment='this username is too short',
+            signature='3' * 128,
+            signing_ts='123'
         )
 
     def test05HideComments(self):
-        comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
-        comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
+        comm = create_comment(
+            comment='Comment #1',
+            claim_id=self.claimId,
+            channel_id='1'*40,
+            channel_name='@Doge123',
+            signature='a'*128,
+            signing_ts='123'
+        )
+        comment = get_comment(comm['comment_id'])
         self.assertFalse(comment['is_hidden'])
-        success = hide_comments_by_id(self.conn, [comm['comment_id']])
+
+        success = set_hidden_flag([comm['comment_id']])
         self.assertTrue(success)
-        comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
+
+        comment = get_comment(comm['comment_id'])
         self.assertTrue(comment['is_hidden'])
-        success = hide_comments_by_id(self.conn, [comm['comment_id']])
+
+        success = set_hidden_flag([comm['comment_id']])
         self.assertTrue(success)
-        comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
+
+        comment = get_comment(comm['comment_id'])
         self.assertTrue(comment['is_hidden'])
 
     def test06DeleteComments(self):
-        comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
-        comments = get_claim_comments(self.conn, self.claimId)
-        match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items']))
-        self.assertTrue(match)
-        deleted = delete_comment_by_id(self.conn, comm['comment_id'])
+        # make sure that the comment was created
+        comm = create_comment(
+            comment='Comment #1',
+            claim_id=self.claimId,
+            channel_id='1'*40,
+            channel_name='@Doge123',
+            signature='a'*128,
+            signing_ts='123'
+        )
+        comments = comment_list(self.claimId)
+        match = [x for x in comments['items'] if x['comment_id'] == comm['comment_id']]
+        self.assertTrue(len(match) > 0)
+
+        deleted = delete_comment(comm['comment_id'])
         self.assertTrue(deleted)
-        comments = get_claim_comments(self.conn, self.claimId)
-        match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items']))
+
+        # make sure that we can't find the comment here
+        comments = comment_list(self.claimId)
+        match = [x for x in comments['items'] if x['comment_id'] == comm['comment_id']]
         self.assertFalse(match)
-        deleted = delete_comment_by_id(self.conn, comm['comment_id'])
-        self.assertFalse(deleted)
+        self.assertRaises(
+            ValueError,
+            delete_comment,
+            comment_id=comm['comment_id'],
+        )
 
 
 class ListDatabaseTest(DatabaseTestCase):
@@ -169,61 +195,75 @@ class ListDatabaseTest(DatabaseTestCase):
     def testLists(self):
         for claim_id in self.claim_ids:
             with self.subTest(claim_id=claim_id):
-                comments = get_claim_comments(self.conn, claim_id)
+                comments = comment_list(claim_id)
                 self.assertIsNotNone(comments)
                 self.assertGreater(comments['page_size'], 0)
                 self.assertIn('has_hidden_comments', comments)
                 self.assertFalse(comments['has_hidden_comments'])
-                top_comments = get_claim_comments(self.conn, claim_id, top_level=True, page=1, page_size=50)
+                top_comments = comment_list(claim_id, top_level=True, page=1, page_size=50)
                 self.assertIsNotNone(top_comments)
                 self.assertEqual(top_comments['page_size'], 50)
                 self.assertEqual(top_comments['page'], 1)
                 self.assertGreaterEqual(top_comments['total_pages'], 0)
                 self.assertGreaterEqual(top_comments['total_items'], 0)
-                comment_ids = get_comment_ids(self.conn, claim_id, page_size=50, page=1)
+                comment_ids = comment_list(claim_id, page_size=50, page=1)
                 with self.subTest(comment_ids=comment_ids):
                     self.assertIsNotNone(comment_ids)
                     self.assertLessEqual(len(comment_ids), 50)
-                    matching_comments = get_comments_by_id(self.conn, comment_ids)
+                    matching_comments = (comment_ids)
                     self.assertIsNotNone(matching_comments)
                     self.assertEqual(len(matching_comments), len(comment_ids))
 
     def testHiddenCommentLists(self):
         claim_id = 'a'*40
-        comm1 = create_comment_or_error(self.conn, 'Comment #1', claim_id, '1'*40, '@Doge123', 'a'*128, '123')
-        comm2 = create_comment_or_error(self.conn, 'Comment #2', claim_id, '1'*40, '@Doge123', 'b'*128, '123')
-        comm3 = create_comment_or_error(self.conn, 'Comment #3', claim_id, '1'*40, '@Doge123', 'c'*128, '123')
+        comm1 = create_comment(
+            'Comment #1',
+            claim_id,
+            channel_id='1'*40,
+            channel_name='@Doge123',
+            signature='a'*128,
+            signing_ts='123'
+        )
+        comm2 = create_comment(
+            'Comment #2', claim_id,
+            channel_id='1'*40,
+            channel_name='@Doge123',
+            signature='b'*128,
+            signing_ts='123'
+        )
+        comm3 = create_comment(
+            'Comment #3', claim_id,
+            channel_id='1'*40,
+            channel_name='@Doge123',
+            signature='c'*128,
+            signing_ts='123'
+        )
         comments = [comm1, comm2, comm3]
 
-        comment_list = get_claim_comments(self.conn, claim_id)
-        self.assertIn('items', comment_list)
-        self.assertIn('has_hidden_comments', comment_list)
-        self.assertEqual(len(comments), comment_list['total_items'])
-        self.assertIn('has_hidden_comments', comment_list)
-        self.assertFalse(comment_list['has_hidden_comments'])
-        hide_comments_by_id(self.conn, [comm2['comment_id']])
+        listed_comments = comment_list(claim_id)
+        self.assertEqual(len(comments), listed_comments['total_items'])
+        self.assertFalse(listed_comments['has_hidden_comments'])
 
-        default_comments = get_claim_hidden_comments(self.conn, claim_id)
-        self.assertIn('has_hidden_comments', default_comments)
+        set_hidden_flag([comm2['comment_id']])
+        hidden = comment_list(claim_id, exclude_mode='hidden')
 
-        hidden_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=True)
-        self.assertIn('has_hidden_comments', hidden_comments)
-        self.assertEqual(default_comments, hidden_comments)
+        self.assertTrue(hidden['has_hidden_comments'])
+        self.assertGreater(len(hidden['items']), 0)
 
-        hidden_comment = hidden_comments['items'][0]
+        visible = comment_list(claim_id, exclude_mode='visible')
+        self.assertFalse(visible['has_hidden_comments'])
+        self.assertNotEqual(listed_comments['items'], visible['items'])
+
+        # make sure the hidden comment is the one we marked as hidden
+        hidden_comment = hidden['items'][0]
         self.assertEqual(hidden_comment['comment_id'], comm2['comment_id'])
 
-        visible_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=False)
-        self.assertIn('has_hidden_comments', visible_comments)
-        self.assertNotIn(hidden_comment, visible_comments['items'])
-
-        hidden_ids = [c['comment_id'] for c in hidden_comments['items']]
-        visible_ids = [c['comment_id'] for c in visible_comments['items']]
+        hidden_ids = [c['comment_id'] for c in hidden['items']]
+        visible_ids = [c['comment_id'] for c in visible['items']]
         composite_ids = hidden_ids + visible_ids
+        listed_comments = comment_list(claim_id)
+        all_ids = [c['comment_id'] for c in listed_comments['items']]
         composite_ids.sort()
-
-        comment_list = get_claim_comments(self.conn, claim_id)
-        all_ids = [c['comment_id'] for c in comment_list['items']]
         all_ids.sort()
         self.assertEqual(composite_ids, all_ids)
 
diff --git a/test/testcase.py b/test/testcase.py
index ddd8b44..12e1197 100644
--- a/test/testcase.py
+++ b/test/testcase.py
@@ -1,12 +1,39 @@
 import os
 import pathlib
 import unittest
-from asyncio.runners import _cancel_all_tasks  # type: ignore
 from unittest.case import _Outcome
 
 import asyncio
+from asyncio.runners import _cancel_all_tasks  # type: ignore
+from peewee import *
+
+from src.database.models import Channel, Comment
+
+
+test_db = SqliteDatabase(':memory:')
+
+
+MODELS = [Channel, Comment]
+
+
+class DatabaseTestCase(unittest.TestCase):
+    def __init__(self, methodName='DatabaseTest'):
+        super().__init__(methodName)
+
+    def setUp(self) -> None:
+        super().setUp()
+        test_db.bind(MODELS, bind_refs=False, bind_backrefs=False)
+
+        test_db.connect()
+        test_db.create_tables(MODELS)
+
+    def tearDown(self) -> None:
+        # drop tables for next test
+        test_db.drop_tables(MODELS)
+
+        # close connection
+        test_db.close()
 
-from src.database.queries import obtain_connection, setup_database
 
 
 class AsyncioTestCase(unittest.TestCase):
@@ -117,21 +144,3 @@ class AsyncioTestCase(unittest.TestCase):
                     self.loop.run_until_complete(maybe_coroutine)
 
 
-class DatabaseTestCase(unittest.TestCase):
-    db_file = 'test.db'
-
-    def __init__(self, methodName='DatabaseTest'):
-        super().__init__(methodName)
-        if pathlib.Path(self.db_file).exists():
-            os.remove(self.db_file)
-
-    def setUp(self) -> None:
-        super().setUp()
-        setup_database(self.db_file)
-        self.conn = obtain_connection(self.db_file)
-        self.addCleanup(self.conn.close)
-        self.addCleanup(os.remove, self.db_file)
-
-    def tearDown(self) -> None:
-        self.conn.close()
-

From e0b6d16c89eed19f3053bb75a2ac675ba69ed48e Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 30 Mar 2020 22:00:10 -0400
Subject: [PATCH 22/46] Removes certain sqlite-specific functions from
 application

---
 src/server/app.py | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/src/server/app.py b/src/server/app.py
index 446c378..35471fd 100644
--- a/src/server/app.py
+++ b/src/server/app.py
@@ -10,7 +10,7 @@ import aiojobs.aiohttp
 from aiohttp import web
 
 from src.database.queries import obtain_connection, DatabaseWriter
-from src.database.queries import setup_database, backup_database
+from src.database.queries import setup_database
 from src.server.handles import api_endpoint, get_api_endpoint
 
 logger = logging.getLogger(__name__)
@@ -24,21 +24,9 @@ async def setup_db_schema(app):
         logger.info(f'Database already exists in {app["db_path"]}, skipping setup')
 
 
-async def database_backup_routine(app):
-    try:
-        while True:
-            await asyncio.sleep(app['config']['backup_int'])
-            with app['reader'] as conn:
-                logger.debug('backing up database')
-                backup_database(conn, app['backup'])
-    except asyncio.CancelledError:
-        pass
-
-
 async def start_background_tasks(app):
     # Reading the DB
     app['reader'] = obtain_connection(app['db_path'], True)
-    app['waitful_backup'] = asyncio.create_task(database_backup_routine(app))
 
     # Scheduler to prevent multiple threads from writing to DB simulataneously
     app['comment_scheduler'] = await aiojobs.create_scheduler(limit=1, pending_limit=0)
@@ -50,9 +38,6 @@ async def start_background_tasks(app):
 
 
 async def close_database_connections(app):
-    logger.info('Ending background backup loop')
-    app['waitful_backup'].cancel()
-    await app['waitful_backup']
     app['reader'].close()
     app['writer'].close()
     app['db_writer'].cleanup()
@@ -67,7 +52,7 @@ async def close_schedulers(app):
 
 
 class CommentDaemon:
-    def __init__(self, config, db_file=None, backup=None, **kwargs):
+    def __init__(self, config, db_file=None, **kwargs):
         app = web.Application()
 
         # configure the config
@@ -76,8 +61,6 @@ class CommentDaemon:
 
         # configure the db file
         app['db_path'] = db_file or config.get('db_path')
-        if app['db_path']:
-            app['backup'] = backup or '.'.join((app['db_path'], 'backup'))
 
         # configure the order of tasks to run during app lifetime
         app.on_startup.append(setup_db_schema)

From 115371163609db3ea7689c92e31432888388cd34 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 31 Mar 2020 13:55:44 -0400
Subject: [PATCH 23/46] Reimplements API methods using ORM

---
 src/database/models.py |  31 ---------
 src/server/handles.py  | 154 +++++++++++++++++++++++++++++++++--------
 2 files changed, 126 insertions(+), 59 deletions(-)

diff --git a/src/database/models.py b/src/database/models.py
index 0658c33..43a1bbb 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -126,37 +126,6 @@ def get_comment(comment_id: str) -> dict:
             .pop())
 
 
-def get_comment_ids(claim_id: str = None, parent_id: str = None,
-                    page: int = 1, page_size: int = 50, flattened=False) -> dict:
-    results = comment_list(
-        claim_id, parent_id,
-        top_level=(parent_id is None),
-        page=page, page_size=page_size,
-        select_fields=['comment_id', 'parent_id']
-    )
-    if flattened:
-        results.update({
-            'items': [item['comment_id'] for item in results['items']],
-            'replies': [(item['comment_id'], item.get('parent_id')) for item in results['items']]
-        })
-    return results
-
-
-def get_comments_by_id(comment_ids: typing.Union[list, tuple]) -> dict:
-    expression = Comment.comment_id.in_(comment_ids)
-    return comment_list(expressions=expression, page_size=len(comment_ids))
-
-
-def get_channel_from_comment_id(comment_id: str) -> dict:
-    results = comment_list(
-        expressions=(Comment.comment_id == comment_id),
-        select_fields=['channel_name', 'channel_id', 'channel_url'],
-        page_size=1
-    )
-    # todo: make the return type here consistent
-    return results['items'].pop()
-
-
 def create_comment_id(comment: str, channel_id: str, timestamp: int):
     # We convert the timestamp from seconds into minutes
     # to prevent spammers from commenting the same BS everywhere.
diff --git a/src/server/handles.py b/src/server/handles.py
index a63504b..bcc2fa6 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -1,16 +1,22 @@
 import asyncio
 import logging
 import time
+import typing
 
 from aiohttp import web
 from aiojobs.aiohttp import atomic
 
-import src.database.queries as db
-from src.database.writes import abandon_comment, create_comment
-from src.database.writes import hide_comments
-from src.database.writes import edit_comment
-from src.misc import clean_input_params
+from src.server.validation import validate_signature_from_claim
+from src.misc import clean_input_params, get_claim_from_id
 from src.server.errors import make_error, report_error
+from src.database.models import Comment, Channel
+from src.database.models import get_comment
+from src.database.models import comment_list
+from src.database.models import create_comment
+from src.database.models import edit_comment
+from src.database.models import delete_comment
+from src.database.models import set_hidden_flag
+
 
 logger = logging.getLogger(__name__)
 
@@ -20,37 +26,127 @@ def ping(*args):
     return 'pong'
 
 
-def handle_get_channel_from_comment_id(app, kwargs: dict):
-    return db.get_channel_id_from_comment_id(app['reader'], **kwargs)
+def handle_get_channel_from_comment_id(app: web.Application, comment_id: str) -> dict:
+    comment = get_comment(comment_id)
+    return {
+        'channel_id': comment['channel_id'],
+        'channel_name': comment['channel_name']
+    }
 
 
-def handle_get_comment_ids(app, kwargs):
-    return db.get_comment_ids(app['reader'], **kwargs)
+def handle_get_comment_ids(
+        app: web.Application,
+        claim_id: str,
+        parent_id: str = None,
+        page: int = 1,
+        page_size: int = 50,
+        flattened=False
+) -> dict:
+    results = comment_list(
+        claim_id=claim_id,
+        parent_id=parent_id,
+        top_level=(parent_id is None),
+        page=page,
+        page_size=page_size,
+        select_fields=['comment_id', 'parent_id']
+    )
+    if flattened:
+        results.update({
+            'items': [item['comment_id'] for item in results['items']],
+            'replies': [(item['comment_id'], item.get('parent_id'))
+                        for item in results['items']]
+        })
+    return results
 
 
-def handle_get_claim_comments(app, kwargs):
-    return db.get_claim_comments(app['reader'], **kwargs)
+def handle_get_comments_by_id(
+        app: web.Application,
+        comment_ids: typing.Union[list, tuple]
+) -> dict:
+    expression = Comment.comment_id.in_(comment_ids)
+    return comment_list(expressions=expression, page_size=len(comment_ids))
 
 
-def handle_get_comments_by_id(app, kwargs):
-    return db.get_comments_by_id(app['reader'], **kwargs)
+def handle_get_claim_comments(
+        app: web.Application,
+        claim_id: str,
+        parent_id: str = None,
+        page: int = 1,
+        page_size: int = 50,
+        top_level: bool = False
+) -> dict:
+    return comment_list(
+        claim_id=claim_id,
+        parent_id=parent_id,
+        page=page,
+        page_size=page_size,
+        top_level=top_level
+    )
 
 
-def handle_get_claim_hidden_comments(app, kwargs):
-    return db.get_claim_hidden_comments(app['reader'], **kwargs)
+def handle_get_claim_hidden_comments(
+        app: web.Application,
+        claim_id: str,
+        hidden: bool,
+        page: int = 1,
+        page_size: int = 50,
+) -> dict:
+    exclude = 'hidden' if hidden else 'visible'
+    return comment_list(
+        claim_id=claim_id,
+        exclude_mode=exclude,
+        page=page,
+        page_size=page_size
+    )
+
+
+def get_channel_from_comment_id(app, comment_id: str) -> dict:
+    results = comment_list(
+        expressions=(Comment.comment_id == comment_id),
+        select_fields=['channel_name', 'channel_id', 'channel_url'],
+        page_size=1
+    )
+    # todo: make the return type here consistent
+    return results['items'].pop()
 
 
 async def handle_abandon_comment(app, params):
-    return {'abandoned': await abandon_comment(app, **params)}
+    # return {'abandoned': await abandon_comment(app, **params)}
+    raise NotImplementedError
 
 
-async def handle_hide_comments(app, params):
-    return {'hidden': await hide_comments(app, **params)}
+async def handle_hide_comments(app, pieces: list = None, claim_id: str = None) -> dict:
+
+    # return {'hidden': await hide_comments(app, **params)}
+    raise NotImplementedError
 
 
-async def handle_edit_comment(app, params):
-    if await edit_comment(app, **params):
-        return db.get_comment_or_none(app['reader'], params['comment_id'])
+async def handle_edit_comment(app, comment: str = None, comment_id: str = None,
+                              signature: str = None, signing_ts: str = None, **params) -> dict:
+    current = get_comment(comment_id)
+    channel_claim = await get_claim_from_id(app, current['channel_id'])
+    if not validate_signature_from_claim(channel_claim, signature, signing_ts, comment):
+        raise ValueError('Signature could not be validated')
+
+    with app['db'].atomic():
+        if not edit_comment(comment_id, comment, signature, signing_ts):
+            raise ValueError('Comment could not be edited')
+        return get_comment(comment_id)
+
+
+def handle_create_comment(app, comment: str = None, claim_id: str = None,
+                          parent_id: str = None, channel_id: str = None, channel_name: str = None,
+                          signature: str = None, signing_ts: str = None) -> dict:
+    with app['db'].atomic():
+        return create_comment(
+            comment=comment,
+            claim_id=claim_id,
+            parent_id=parent_id,
+            channel_id=channel_id,
+            channel_name=channel_name,
+            signature=signature,
+            signing_ts=signing_ts
+        )
 
 
 METHODS = {
@@ -59,8 +155,8 @@ METHODS = {
     'get_claim_hidden_comments': handle_get_claim_hidden_comments,  # this gets used
     'get_comment_ids': handle_get_comment_ids,
     'get_comments_by_id': handle_get_comments_by_id,    # this gets used
-    'get_channel_from_comment_id': handle_get_channel_from_comment_id,  # this gets used
-    'create_comment': create_comment,   # this gets used
+    'get_channel_from_comment_id': get_channel_from_comment_id,  # this gets used
+    'create_comment': handle_create_comment,   # this gets used
     'delete_comment': handle_abandon_comment,
     'abandon_comment': handle_abandon_comment,  # this gets used
     'hide_comments': handle_hide_comments,  # this gets used
@@ -78,17 +174,19 @@ async def process_json(app, body: dict) -> dict:
         start = time.time()
         try:
             if asyncio.iscoroutinefunction(METHODS[method]):
-                result = await METHODS[method](app, params)
+                result = await METHODS[method](app, **params)
             else:
-                result = METHODS[method](app, params)
-            response['result'] = result
+                result = METHODS[method](app, **params)
+
         except Exception as err:
-            logger.exception(f'Got {type(err).__name__}:')
+            logger.exception(f'Got {type(err).__name__}:\n{err}')
             if type(err) in (ValueError, TypeError):  # param error, not too important
                 response['error'] = make_error('INVALID_PARAMS', err)
             else:
                 response['error'] = make_error('INTERNAL', err)
-                await app['webhooks'].spawn(report_error(app, err, body))
+            await app['webhooks'].spawn(report_error(app, err, body))
+        else:
+            response['result'] = result
 
         finally:
             end = time.time()

From bd06d1c992e25361b576489a5638007a9b929753 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 31 Mar 2020 13:57:30 -0400
Subject: [PATCH 24/46] Improves database configuration

---
 setup.py               |  7 ++--
 src/database/models.py | 13 +-------
 src/main.py            | 12 +++++--
 src/server/app.py      | 72 ++++++++++++++++++++++--------------------
 test/test_database.py  |  2 +-
 test/test_server.py    | 26 +++++++--------
 test/testcase.py       |  1 -
 7 files changed, 65 insertions(+), 68 deletions(-)

diff --git a/setup.py b/setup.py
index 25087e4..c8755cf 100644
--- a/setup.py
+++ b/setup.py
@@ -17,12 +17,11 @@ setup(
         'mysql-connector-python',
         'pyyaml',
         'Faker>=1.0.7',
-        'asyncio>=3.4.3',
-        'aiohttp==3.5.4',
-        'aiojobs==0.2.2',
+        'asyncio',
+        'aiohttp',
+        'aiojobs',
         'ecdsa>=0.13.3',
         'cryptography==2.5',
-        'aiosqlite==0.10.0',
         'PyNaCl>=1.3.0',
         'requests',
         'cython',
diff --git a/src/database/models.py b/src/database/models.py
index 43a1bbb..f999f34 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -12,14 +12,6 @@ from src.server.validation import is_valid_base_comment
 from src.misc import clean
 
 
-def get_database_connection(dbms, db_name, **params):
-    if dbms == 'mysql':
-        return MySQLDatabase(db_name, **params)
-    else:
-        # return SqliteDatabase('/home/oleg/PycharmProjects/comment-server/database/default_pw.db')
-        return SqliteDatabase(db_name)
-
-
 class Channel(Model):
     claim_id = TextField(column_name='ClaimId', primary_key=True)
     name = TextField(column_name='Name')
@@ -157,7 +149,7 @@ def create_comment(comment: str = None, claim_id: str = None,
         raise ValueError('Invalid Parameters given for comment')
 
     channel, _ = Channel.get_or_create(name=channel_name, claim_id=channel_id)
-    if parent_id:
+    if parent_id and not claim_id:
         parent: Comment = Comment.get_by_id(parent_id)
         claim_id = parent.claim_id
 
@@ -219,7 +211,4 @@ if __name__ == '__main__':
                      (Comment.claim_id ** '420%'))
     )
 
-    ids = get_comment_ids('4207d2378bf4340e68c9d88faf7ee24ea1a1f95a')
-
     print(json.dumps(comments, indent=4))
-    print(json.dumps(ids, indent=4))
diff --git a/src/main.py b/src/main.py
index c22a82b..124c7d6 100644
--- a/src/main.py
+++ b/src/main.py
@@ -84,12 +84,13 @@ def get_config(filepath):
 
 
 def setup_db_from_config(config: dict):
-    if 'sqlite' in config['database']:
+    mode = config['mode']
+    if config[mode]['database'] == 'sqlite':
         if not os.path.exists(DATABASE_DIR):
             os.mkdir(DATABASE_DIR)
 
-        config['db_path'] = os.path.join(
-            DATABASE_DIR, config['database']['sqlite']
+        config[mode]['db_file'] = os.path.join(
+            DATABASE_DIR, config[mode]['name']
         )
 
 
@@ -98,10 +99,15 @@ def main(argv=None):
     parser = argparse.ArgumentParser(description='LBRY Comment Server')
     parser.add_argument('--port', type=int)
     parser.add_argument('--config', type=str)
+    parser.add_argument('--mode', type=str)
     args = parser.parse_args(argv)
 
     config = get_config(CONFIG_FILE) if not args.config else args.config
     setup_logging_from_config(config)
+
+    if args.mode:
+        config['mode'] = args.mode
+
     setup_db_from_config(config)
 
     if args.port:
diff --git a/src/server/app.py b/src/server/app.py
index 35471fd..389083e 100644
--- a/src/server/app.py
+++ b/src/server/app.py
@@ -1,7 +1,6 @@
 # cython: language_level=3
 import asyncio
 import logging
-import pathlib
 import signal
 import time
 
@@ -9,61 +8,67 @@ import aiojobs
 import aiojobs.aiohttp
 from aiohttp import web
 
-from src.database.queries import obtain_connection, DatabaseWriter
-from src.database.queries import setup_database
+from peewee import *
 from src.server.handles import api_endpoint, get_api_endpoint
+from src.database.models import Comment, Channel
 
+MODELS = [Comment, Channel]
 logger = logging.getLogger(__name__)
 
 
-async def setup_db_schema(app):
-    if not pathlib.Path(app['db_path']).exists():
-        logger.info(f'Setting up schema in {app["db_path"]}')
-        setup_database(app['db_path'])
-    else:
-        logger.info(f'Database already exists in {app["db_path"]}, skipping setup')
+def setup_database(app):
+    config = app['config']
+    mode = config['mode']
+
+    # switch between Database objects
+    if config[mode]['database'] == 'mysql':
+        app['db'] = MySQLDatabase(
+            database=config[mode]['name'],
+            user=config[mode]['user'],
+            host=config[mode]['host'],
+            password=config[mode]['password'],
+            port=config[mode]['port'],
+        )
+    elif config[mode]['database'] == 'sqlite':
+        app['db'] = SqliteDatabase(
+            config[mode]['file'],
+            pragmas=config[mode]['pragmas']
+        )
+
+    # bind the Model list to the database
+    app['db'].bind(MODELS, bind_refs=False, bind_backrefs=False)
 
 
 async def start_background_tasks(app):
-    # Reading the DB
-    app['reader'] = obtain_connection(app['db_path'], True)
-
-    # Scheduler to prevent multiple threads from writing to DB simulataneously
-    app['comment_scheduler'] = await aiojobs.create_scheduler(limit=1, pending_limit=0)
-    app['db_writer'] = DatabaseWriter(app['db_path'])
-    app['writer'] = app['db_writer'].connection
+    app['db'].connect()
+    app['db'].create_tables(MODELS)
 
     # for requesting to external and internal APIs
     app['webhooks'] = await aiojobs.create_scheduler(pending_limit=0)
 
 
 async def close_database_connections(app):
-    app['reader'].close()
-    app['writer'].close()
-    app['db_writer'].cleanup()
+    app['db'].close()
 
 
 async def close_schedulers(app):
-    logger.info('Closing comment_scheduler')
-    await app['comment_scheduler'].close()
-
     logger.info('Closing scheduler for webhook requests')
     await app['webhooks'].close()
 
 
 class CommentDaemon:
-    def __init__(self, config, db_file=None, **kwargs):
+    def __init__(self, config, **kwargs):
         app = web.Application()
+        app['config'] = config
 
         # configure the config
-        app['config'] = config
-        self.config = app['config']
+        self.config = config
+        self.host = config['host']
+        self.port = config['port']
 
-        # configure the db file
-        app['db_path'] = db_file or config.get('db_path')
+        setup_database(app)
 
         # configure the order of tasks to run during app lifetime
-        app.on_startup.append(setup_db_schema)
         app.on_startup.append(start_background_tasks)
         app.on_shutdown.append(close_schedulers)
         app.on_cleanup.append(close_database_connections)
@@ -85,20 +90,19 @@ class CommentDaemon:
         await self.app_runner.setup()
         self.app_site = web.TCPSite(
             runner=self.app_runner,
-            host=host or self.config['host'],
-            port=port or self.config['port'],
+            host=host or self.host,
+            port=port or self.port,
         )
         await self.app_site.start()
-        logger.info(f'Comment Server is running on {self.config["host"]}:{self.config["port"]}')
+        logger.info(f'Comment Server is running on {self.host}:{self.port}')
 
     async def stop(self):
         await self.app_runner.shutdown()
         await self.app_runner.cleanup()
 
 
-def run_app(config, db_file=None):
-    comment_app = CommentDaemon(config=config, db_file=db_file, close_timeout=5.0)
-
+def run_app(config):
+    comment_app = CommentDaemon(config=config)
     loop = asyncio.get_event_loop()
 
     def __exit():
diff --git a/test/test_database.py b/test/test_database.py
index bf25bba..c698ad1 100644
--- a/test/test_database.py
+++ b/test/test_database.py
@@ -6,7 +6,7 @@ from faker.providers import misc
 
 from src.database.models import create_comment
 from src.database.models import delete_comment
-from src.database.models import comment_list, get_comment, get_comments_by_id
+from src.database.models import comment_list, get_comment
 from src.database.models import set_hidden_flag
 from test.testcase import DatabaseTestCase
 
diff --git a/test/test_server.py b/test/test_server.py
index 55fcdba..56bccfb 100644
--- a/test/test_server.py
+++ b/test/test_server.py
@@ -17,6 +17,8 @@ from test.testcase import AsyncioTestCase
 
 
 config = get_config(CONFIG_FILE)
+config['mode'] = 'testing'
+config['testing']['file'] = ':memory:'
 
 
 if 'slack_webhook' in config:
@@ -74,10 +76,10 @@ def create_test_comments(values: iter, **default):
 
 
 class ServerTest(AsyncioTestCase):
-    db_file = 'test.db'
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        config['mode'] = 'testing'
+        config['testing']['file'] = ':memory:'
         self.host = 'localhost'
         self.port = 5931
 
@@ -88,11 +90,10 @@ class ServerTest(AsyncioTestCase):
     @classmethod
     def tearDownClass(cls) -> None:
         print('exit reached')
-        os.remove(cls.db_file)
 
     async def asyncSetUp(self):
         await super().asyncSetUp()
-        self.server = app.CommentDaemon(config, db_file=self.db_file)
+        self.server = app.CommentDaemon(config)
         await self.server.start(host=self.host, port=self.port)
         self.addCleanup(self.server.stop)
 
@@ -138,14 +139,16 @@ class ServerTest(AsyncioTestCase):
         test_all = create_test_comments(replace.keys(), **{
             k: None for k in replace.keys()
         })
+        test_all.reverse()
         for test in test_all:
-            with self.subTest(test=test):
+            nulls = 'null fields: ' + ', '.join(k for k, v in test.items() if not v)
+            with self.subTest(test=nulls):
                 message = await self.post_comment(**test)
                 self.assertTrue('result' in message or 'error' in message)
                 if 'error' in message:
-                    self.assertFalse(is_valid_base_comment(**test))
+                    self.assertFalse(is_valid_base_comment(**test, strict=True))
                 else:
-                    self.assertTrue(is_valid_base_comment(**test))
+                    self.assertTrue(is_valid_base_comment(**test, strict=True))
 
     async def test04CreateAllReplies(self):
         claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8'
@@ -223,7 +226,8 @@ class ListCommentsTest(AsyncioTestCase):
         super().__init__(*args, **kwargs)
         self.host = 'localhost'
         self.port = 5931
-        self.db_file = 'list_test.db'
+        config['mode'] = 'testing'
+        config['testing']['file'] = ':memory:'
         self.claim_id = '1d8a5cc39ca02e55782d619e67131c0a20843be8'
         self.comment_ids = None
 
@@ -234,10 +238,6 @@ class ListCommentsTest(AsyncioTestCase):
     async def post_comment(self, **params):
         return await jsonrpc_post(self.url, 'create_comment', **params)
 
-    def tearDown(self) -> None:
-        print('exit reached')
-        os.remove(self.db_file)
-
     async def create_lots_of_comments(self, n=23):
         self.comment_list = [{key: self.replace[key]() for key in self.replace.keys()} for _ in range(23)]
         for comment in self.comment_list:
@@ -247,7 +247,7 @@ class ListCommentsTest(AsyncioTestCase):
 
     async def asyncSetUp(self):
         await super().asyncSetUp()
-        self.server = app.CommentDaemon(config, db_file=self.db_file)
+        self.server = app.CommentDaemon(config)
         await self.server.start(self.host, self.port)
         self.addCleanup(self.server.stop)
 
diff --git a/test/testcase.py b/test/testcase.py
index 12e1197..415041d 100644
--- a/test/testcase.py
+++ b/test/testcase.py
@@ -35,7 +35,6 @@ class DatabaseTestCase(unittest.TestCase):
         test_db.close()
 
 
-
 class AsyncioTestCase(unittest.TestCase):
     # Implementation inspired by discussion:
     #  https://bugs.python.org/issue32972

From a6f056821f719d6c52a502c1f0ea050cb2c56f7d Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 31 Mar 2020 13:59:20 -0400
Subject: [PATCH 25/46] Adds a `strict` validator which fails upon possible FK
 violations

---
 src/server/validation.py | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/src/server/validation.py b/src/server/validation.py
index ef1c841..3b251ba 100644
--- a/src/server/validation.py
+++ b/src/server/validation.py
@@ -51,11 +51,31 @@ def claim_id_is_valid(claim_id: str) -> bool:
 
 
 # default to None so params can be treated as kwargs; param count becomes more manageable
-def is_valid_base_comment(comment: str = None, claim_id: str = None, parent_id: str = None, **kwargs) -> bool:
-    return comment and body_is_valid(comment) and \
-           ((claim_id and claim_id_is_valid(claim_id)) or  # parentid is used in place of claimid in replies
-            (parent_id and comment_id_is_valid(parent_id))) \
-           and is_valid_credential_input(**kwargs)
+def is_valid_base_comment(
+        comment: str = None,
+        claim_id: str = None,
+        parent_id: str = None,
+        strict: bool = False,
+        **kwargs,
+) -> bool:
+    try:
+        assert comment and body_is_valid(comment)
+        # strict mode assumes that the parent_id might not exist
+        if strict:
+            assert claim_id and claim_id_is_valid(claim_id)
+            assert parent_id is None or comment_id_is_valid(parent_id)
+        # non-strict removes reference restrictions
+        else:
+            assert claim_id or parent_id
+            if claim_id:
+                assert claim_id_is_valid(claim_id)
+            else:
+                assert comment_id_is_valid(parent_id)
+
+    except AssertionError:
+        return False
+    else:
+        return is_valid_credential_input(**kwargs)
 
 
 def is_valid_credential_input(channel_id: str = None, channel_name: str = None,

From 8f12d997aeae2cf2eb9041f241eebe5971fbb039 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 31 Mar 2020 13:59:37 -0400
Subject: [PATCH 26/46] json -> yaml

---
 src/definitions.py | 2 +-
 src/main.py        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/definitions.py b/src/definitions.py
index 9972e13..51654e1 100644
--- a/src/definitions.py
+++ b/src/definitions.py
@@ -2,6 +2,6 @@ import os
 
 SRC_DIR = os.path.dirname(os.path.abspath(__file__))
 ROOT_DIR = os.path.dirname(SRC_DIR)
-CONFIG_FILE = os.path.join(ROOT_DIR, 'config', 'conf.json')
+CONFIG_FILE = os.path.join(ROOT_DIR, 'config', 'conf.yml')
 LOGGING_DIR = os.path.join(ROOT_DIR, 'logs')
 DATABASE_DIR = os.path.join(ROOT_DIR, 'database')
diff --git a/src/main.py b/src/main.py
index 124c7d6..817dac4 100644
--- a/src/main.py
+++ b/src/main.py
@@ -79,7 +79,7 @@ def setup_logging_from_config(conf: dict):
 
 def get_config(filepath):
     with open(filepath, 'r') as cfile:
-        config = yaml.load(cfile, Loader=yaml.FullLoader)
+        config = yaml.load(cfile, Loader=yaml.Loader)
     return config
 
 

From c852697c947f3efb0918bd0d0cdad82465ae9272 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Wed, 1 Apr 2020 18:12:46 -0400
Subject: [PATCH 27/46] reimplements `abandon` method using ORM; no longer
 require `channel_id` param

---
 src/database/models.py |  9 ++++++---
 src/database/writes.py |  4 ++++
 src/server/handles.py  | 43 +++++++++++++++++++++++++++++++++++++++---
 3 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/src/database/models.py b/src/database/models.py
index f999f34..73f8f17 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -113,9 +113,12 @@ def comment_list(claim_id: str = None, parent_id: str = None,
 
 
 def get_comment(comment_id: str) -> dict:
-    return (comment_list(expressions=(Comment.comment_id == comment_id), page_size=1)
-            .get('items')
-            .pop())
+    try:
+        comment = comment_list(expressions=(Comment.comment_id == comment_id), page_size=1).get('items').pop()
+    except IndexError:
+        raise ValueError(f'Comment does not exist with id {comment_id}')
+    else:
+        return comment
 
 
 def create_comment_id(comment: str, channel_id: str, timestamp: int):
diff --git a/src/database/writes.py b/src/database/writes.py
index 9f13f3f..61caa78 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -84,13 +84,17 @@ async def hide_comments(app, pieces: list) -> list:
     # TODO: Amortize this process
     claims = {}
     comments_to_hide = []
+    # go through a list of dict objects
     for p in pieces:
+        # maps the comment_id from the piece to a claim_id
         claim_id = comment_cids[p['comment_id']]
+        # resolve the claim from its id
         if claim_id not in claims:
             claim = await get_claim_from_id(app, claim_id)
             if claim:
                 claims[claim_id] = claim
 
+        # get the claim's signing channel, then use it to validate the hidden comment
         channel = claims[claim_id].get('signing_channel')
         if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']):
             comments_to_hide.append(p)
diff --git a/src/server/handles.py b/src/server/handles.py
index bcc2fa6..373fc74 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -5,6 +5,7 @@ import typing
 
 from aiohttp import web
 from aiojobs.aiohttp import atomic
+from peewee import DoesNotExist
 
 from src.server.validation import validate_signature_from_claim
 from src.misc import clean_input_params, get_claim_from_id
@@ -110,9 +111,45 @@ def get_channel_from_comment_id(app, comment_id: str) -> dict:
     return results['items'].pop()
 
 
-async def handle_abandon_comment(app, params):
-    # return {'abandoned': await abandon_comment(app, **params)}
-    raise NotImplementedError
+async def handle_abandon_comment(
+        app: web.Application,
+        comment_id: str,
+        signature: str,
+        signing_ts: str,
+        **kwargs,
+) -> dict:
+    comment = get_comment(comment_id)
+    try:
+        channel = await get_claim_from_id(app, comment['channel_id'])
+    except DoesNotExist:
+        raise ValueError('Could not find a channel associated with the given comment')
+    else:
+        if not validate_signature_from_claim(channel, signature, signing_ts, comment_id):
+            raise ValueError('Abandon signature could not be validated')
+
+    with app['db'].atomic():
+        return {
+            'abandoned': delete_comment(comment_id)
+        }
+
+
+async def handle_hide_comments(app: web.Application, pieces: list, hide: bool = True) -> dict:
+    # let's get all the distinct claim_ids from the list of comment_ids
+    pieces_by_id = {p['comment_id']: p for p in pieces}
+    comment_ids = list(pieces_by_id.keys())
+    comments = (Comment
+                .select(Comment.comment_id, Comment.claim_id)
+                .where(Comment.comment_id.in_(comment_ids))
+                .tuples())
+
+    # resolve the claims and map them to their corresponding comment_ids
+    claims = {}
+    for comment_id, claim_id in comments:
+        try:
+            # try and resolve the claim, if fails then we mark it as null
+            # and remove the associated comment from the pieces
+            if claim_id not in claims:
+                claims[claim_id] = await get_claim_from_id(app, claim_id)
 
 
 async def handle_hide_comments(app, pieces: list = None, claim_id: str = None) -> dict:

From 08060a71d3b1a183536c3d38190e6d10fdfc0ac3 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Wed, 1 Apr 2020 18:13:28 -0400
Subject: [PATCH 28/46] Implements `hide_comments` method using ORM

---
 src/server/handles.py | 32 +++++++++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/src/server/handles.py b/src/server/handles.py
index 373fc74..6857908 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -151,11 +151,37 @@ async def handle_hide_comments(app: web.Application, pieces: list, hide: bool =
             if claim_id not in claims:
                 claims[claim_id] = await get_claim_from_id(app, claim_id)
 
+            # try to get a public key to validate
+            if claims[claim_id] is None or 'signing_channel' not in claims[claim_id]:
+                raise ValueError(f'could not get signing channel from claim_id: {claim_id}')
 
-async def handle_hide_comments(app, pieces: list = None, claim_id: str = None) -> dict:
+            # try to validate signature
+            else:
+                channel = claims[claim_id]['signing_channel']
+                piece = pieces_by_id[comment_id]
+                is_valid_signature = validate_signature_from_claim(
+                        claim=channel,
+                        signature=piece['signature'],
+                        signing_ts=piece['signing_ts'],
+                        data=piece['comment_id']
+                )
+                if not is_valid_signature:
+                    raise ValueError(f'could not validate signature on comment_id: {comment_id}')
 
-    # return {'hidden': await hide_comments(app, **params)}
-    raise NotImplementedError
+        except ValueError:
+            # remove the piece from being hidden
+            pieces_by_id.pop(comment_id)
+
+    # remaining items in pieces_by_id have been able to successfully validate
+    with app['db'].atomic():
+        set_hidden_flag(list(pieces_by_id.keys()), hidden=hide)
+
+    query = Comment.select().where(Comment.comment_id.in_(comment_ids)).objects()
+    result = {
+        'hidden': [c.comment_id for c in query if c.is_hidden],
+        'visible': [c.comment_id for c in query if not c.is_hidden],
+    }
+    return result
 
 
 async def handle_edit_comment(app, comment: str = None, comment_id: str = None,

From 20f9ccc8c53fb11f41166a29c39e56c5dedaddaf Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Wed, 1 Apr 2020 18:49:03 -0400
Subject: [PATCH 29/46] Moves to using `CharField` in place of `TextField`

---
 src/database/models.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/database/models.py b/src/database/models.py
index 73f8f17..475d167 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -13,15 +13,15 @@ from src.misc import clean
 
 
 class Channel(Model):
-    claim_id = TextField(column_name='ClaimId', primary_key=True)
-    name = TextField(column_name='Name')
+    claim_id = CharField(column_name='ClaimId', primary_key=True, max_length=40)
+    name = CharField(column_name='Name', max_length=256)
 
     class Meta:
         table_name = 'CHANNEL'
 
 
 class Comment(Model):
-    comment = TextField(column_name='Body')
+    comment = CharField(column_name='Body', max_length=2000)
     channel = ForeignKeyField(
         backref='comments',
         column_name='ChannelId',
@@ -29,9 +29,9 @@ class Comment(Model):
         model=Channel,
         null=True
     )
-    comment_id = TextField(column_name='CommentId', primary_key=True)
+    comment_id = CharField(column_name='CommentId', primary_key=True, max_length=64)
     is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")])
-    claim_id = TextField(column_name='LbryClaimId')
+    claim_id = CharField(max_length=40, column_name='LbryClaimId')
     parent = ForeignKeyField(
         column_name='ParentId',
         field='comment_id',
@@ -39,7 +39,7 @@ class Comment(Model):
         null=True,
         backref='replies'
     )
-    signature = TextField(column_name='Signature', null=True, unique=True)
+    signature = CharField(max_length=128, column_name='Signature', null=True, unique=True)
     signing_ts = TextField(column_name='SigningTs', null=True)
     timestamp = IntegerField(column_name='Timestamp')
 

From 3b6b05200086ea0fc44c4f16619ce5636dc25faf Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Wed, 1 Apr 2020 18:49:37 -0400
Subject: [PATCH 30/46] Removes incorrect method for
 `get_channel_from_comment_id`

---
 src/server/handles.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/src/server/handles.py b/src/server/handles.py
index 6857908..10d2dda 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -101,16 +101,6 @@ def handle_get_claim_hidden_comments(
     )
 
 
-def get_channel_from_comment_id(app, comment_id: str) -> dict:
-    results = comment_list(
-        expressions=(Comment.comment_id == comment_id),
-        select_fields=['channel_name', 'channel_id', 'channel_url'],
-        page_size=1
-    )
-    # todo: make the return type here consistent
-    return results['items'].pop()
-
-
 async def handle_abandon_comment(
         app: web.Application,
         comment_id: str,
@@ -218,7 +208,7 @@ METHODS = {
     'get_claim_hidden_comments': handle_get_claim_hidden_comments,  # this gets used
     'get_comment_ids': handle_get_comment_ids,
     'get_comments_by_id': handle_get_comments_by_id,    # this gets used
-    'get_channel_from_comment_id': get_channel_from_comment_id,  # this gets used
+    'get_channel_from_comment_id': handle_get_channel_from_comment_id,  # this gets used
     'create_comment': handle_create_comment,   # this gets used
     'delete_comment': handle_abandon_comment,
     'abandon_comment': handle_abandon_comment,  # this gets used

From 84cafd643fad0b569d2a3f952cd3e0d441c3f673 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Wed, 1 Apr 2020 18:50:09 -0400
Subject: [PATCH 31/46] replace oracle driver with `pymysql` driver

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c8755cf..f7a397f 100644
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,7 @@ setup(
     data_files=[('config', ['config/conf.json',])],
     include_package_data=True,
     install_requires=[
-        'mysql-connector-python',
+        'pymysql',
         'pyyaml',
         'Faker>=1.0.7',
         'asyncio',

From c27baa89fe79898b6b5b46ded2542dfbf3422abe Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Wed, 1 Apr 2020 18:52:36 -0400
Subject: [PATCH 32/46] rate limit comments

---
 src/database/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/database/models.py b/src/database/models.py
index 475d167..9c40586 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -124,7 +124,7 @@ def get_comment(comment_id: str) -> dict:
 def create_comment_id(comment: str, channel_id: str, timestamp: int):
     # We convert the timestamp from seconds into minutes
     # to prevent spammers from commenting the same BS everywhere.
-    nearest_minute = str(math.floor(timestamp))
+    nearest_minute = str(math.floor(timestamp / 60))
 
     # don't use claim_id for the comment_id anymore so comments
     # are not unique to just one claim

From fd05ea1145bed5389d962350b39e9b1ba21561d3 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 3 Apr 2020 15:33:23 -0400
Subject: [PATCH 33/46] Update travis config & add generic yaml config

---
 .travis.yml     | 23 ++++++++++++++++++++++-
 config/conf.yml | 30 ++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 config/conf.yml

diff --git a/.travis.yml b/.travis.yml
index ac2c250..fe81b23 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,28 @@
 sudo: required
 language: python
 dist: xenial
-python: 3.7
+python: 3.8
+
+# for docker-compose
+services:
+  - docker
+
+# to avoid "well it works on my computer" moments
+env:
+  - DOCKER_COMPOSE_VERSION=1.25.4
+
+before_install:
+  # ensure docker-compose version is as specified above
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
+  # refresh docker images
+  - sudo apt-get update
+
+before_script:
+  - docker-compose up -d
+
 
 jobs:
   include:
diff --git a/config/conf.yml b/config/conf.yml
new file mode 100644
index 0000000..6d90446
--- /dev/null
+++ b/config/conf.yml
@@ -0,0 +1,30 @@
+---
+# for running local-tests without using MySQL for now
+testing:
+  database: sqlite
+  file: comments.db
+  pragmas:
+    journal_mode: wal
+    cache_size: 64000
+    foreign_keys: 0
+    ignore_check_constraints: 1
+    synchronous: 0
+
+# actual database should be running MySQL
+production:
+  database: mysql
+  name: lbry
+  user: lbry
+  password: lbry
+  host: localhost
+  port: 3306
+
+mode: production
+logging:
+  format: "%(asctime)s | %(levelname)s | %(name)s | %(module)s.%(funcName)s:%(lineno)d
+    | %(message)s"
+  aiohttp_format: "%(asctime)s | %(levelname)s | %(name)s | %(message)s"
+  datefmt: "%Y-%m-%d %H:%M:%S"
+host: localhost
+port: 5921
+lbrynet: http://localhost:5279
\ No newline at end of file

From a84e0e0f84d344b512f63a10db4db033d217ccbd Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 3 Apr 2020 15:38:29 -0400
Subject: [PATCH 34/46] Add docker-compose.yml

---
 docker-compose.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 docker-compose.yml

diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..3faf5f2
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,24 @@
+version: "3.7"
+services:
+  ###########
+  ## MySQL ##
+  ###########
+  mysql:
+    image: mysql/mysql-server:5.7.27
+    restart: "no"
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=true
+      - MYSQL_DATABASE=lbry
+      - MYSQL_USER=lbry
+      - MYSQL_PASSWORD=lbry
+      - MYSQL_LOG_CONSOLE=true
+ #############
+  ## Adminer ##
+  #############
+  adminer:
+    image: adminer
+    restart: always
+    ports:
+      - 8080:8080

From 38e9af24ecf5e31b9c76322197ee87e67110b202 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 3 Apr 2020 16:07:23 -0400
Subject: [PATCH 35/46] Update .gitignore

---
 .gitignore | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index c7e09d7..54e4315 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
-config/conf.json
+config/conf.yml
+docker-compose.yml
+

From 75c8f82072b963f4fe7263013f3e69f87c983967 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 3 Apr 2020 16:40:05 -0400
Subject: [PATCH 36/46] Add todos

---
 src/server/handles.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server/handles.py b/src/server/handles.py
index 10d2dda..1c56663 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -187,6 +187,7 @@ async def handle_edit_comment(app, comment: str = None, comment_id: str = None,
         return get_comment(comment_id)
 
 
+# TODO: retrieve stake amounts for each channel & store in db
 def handle_create_comment(app, comment: str = None, claim_id: str = None,
                           parent_id: str = None, channel_id: str = None, channel_name: str = None,
                           signature: str = None, signing_ts: str = None) -> dict:

From d25e03d853a585e5f9ac816f4ebdc020649a2636 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 6 Apr 2020 19:13:12 -0400
Subject: [PATCH 37/46] database name gets set to `social`

---
 config/conf.yml    | 2 +-
 docker-compose.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/conf.yml b/config/conf.yml
index 6d90446..b311ef1 100644
--- a/config/conf.yml
+++ b/config/conf.yml
@@ -13,7 +13,7 @@ testing:
 # actual database should be running MySQL
 production:
   database: mysql
-  name: lbry
+  name: social
   user: lbry
   password: lbry
   host: localhost
diff --git a/docker-compose.yml b/docker-compose.yml
index 3faf5f2..41530f7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,7 +10,7 @@ services:
       - "3306:3306"
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=true
-      - MYSQL_DATABASE=lbry
+      - MYSQL_DATABASE=social
       - MYSQL_USER=lbry
       - MYSQL_PASSWORD=lbry
       - MYSQL_LOG_CONSOLE=true

From be45a70c362a4bc55bdca143769c044c3a6a2afe Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 6 Apr 2020 19:15:54 -0400
Subject: [PATCH 38/46] sets column names to be lowercase, uses utf8mb4
 charset, utf8mb4_unicode_ci collation

---
 config/conf.yml            |  1 +
 database/default_after.sql | 48 ++++++++++++++++++++++++++++++++++++++
 docker-compose.yml         |  1 +
 src/database/models.py     | 16 ++++++++-----
 src/misc.py                |  2 +-
 src/server/app.py          |  1 +
 6 files changed, 62 insertions(+), 7 deletions(-)
 create mode 100644 database/default_after.sql

diff --git a/config/conf.yml b/config/conf.yml
index b311ef1..fc22322 100644
--- a/config/conf.yml
+++ b/config/conf.yml
@@ -12,6 +12,7 @@ testing:
 
 # actual database should be running MySQL
 production:
+  charset: utf8mb4
   database: mysql
   name: social
   user: lbry
diff --git a/database/default_after.sql b/database/default_after.sql
new file mode 100644
index 0000000..1ee19f4
--- /dev/null
+++ b/database/default_after.sql
@@ -0,0 +1,48 @@
+USE `social`;
+ALTER DATABASE `social`
+    DEFAULT CHARACTER SET utf8mb4
+    DEFAULT COLLATE utf8mb4_unicode_ci;
+
+DROP TABLE IF EXISTS `CHANNEL`;
+CREATE TABLE `CHANNEL` (
+        `claimid` VARCHAR(40)  NOT NULL,
+        -- i cant tell if max name length is 255 or 256
+        `name`  VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+        CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`)
+    )
+CHARACTER SET utf8mb4
+COLLATE utf8mb4_unicode_ci;
+
+DROP TABLE IF EXISTS `COMMENT`;
+CREATE TABLE `COMMENT` (
+        `commentid`   VARCHAR(64)   NOT NULL,
+        `lbryclaimid` VARCHAR(40)   NOT NULL,
+        `channelid`   VARCHAR(40)              DEFAULT NULL,
+        `body`        VARCHAR(5000)
+            CHARACTER SET utf8mb4
+            COLLATE utf8mb4_unicode_ci
+            NOT NULL,
+        `parentid`    VARCHAR(64)               DEFAULT NULL,
+        `signature`   VARCHAR(128)              DEFAULT NULL,
+        `signingts`   VARCHAR(22)               DEFAULT NULL,
+
+        `timestamp`   INTEGER NOT NULL,
+        -- there's no way that the timestamp will ever reach 22 characters
+        `ishidden`    BOOLEAN                    DEFAULT FALSE,
+        CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`),
+        CONSTRAINT `comment_signature_sk` UNIQUE (`signature`),
+        CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`)
+            ON DELETE CASCADE ON UPDATE CASCADE,
+        CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`)
+            ON UPDATE CASCADE ON DELETE CASCADE, -- setting null implies comment is top level
+        CONSTRAINT `channel_signature`
+            CHECK ( `signature` IS NOT NULL AND `signingts` IS NOT NULL)
+    )
+CHARACTER SET utf8mb4
+COLLATE utf8mb4_unicode_ci;
+
+CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`);
+CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`);
+
+
+
diff --git a/docker-compose.yml b/docker-compose.yml
index 41530f7..e5f9099 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,6 +6,7 @@ services:
   mysql:
     image: mysql/mysql-server:5.7.27
     restart: "no"
+    command: --character_set_server=utf8mb4 --max_allowed_packet=1073741824
     ports:
       - "3306:3306"
     environment:
diff --git a/src/database/models.py b/src/database/models.py
index 9c40586..9e72d33 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -13,24 +13,28 @@ from src.misc import clean
 
 
 class Channel(Model):
-    claim_id = CharField(column_name='ClaimId', primary_key=True, max_length=40)
-    name = CharField(column_name='Name', max_length=256)
+    claim_id = CharField(column_name='claimid', primary_key=True, max_length=40)
+    name = CharField(column_name='name', max_length=256)
 
     class Meta:
         table_name = 'CHANNEL'
 
 
 class Comment(Model):
-    comment = CharField(column_name='Body', max_length=2000)
+    comment = CharField(
+        column_name='body',
+        max_length=5000,
+
+    )
     channel = ForeignKeyField(
         backref='comments',
-        column_name='ChannelId',
+        column_name='channelid',
         field='claim_id',
         model=Channel,
         null=True
     )
-    comment_id = CharField(column_name='CommentId', primary_key=True, max_length=64)
-    is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")])
+    comment_id = CharField(column_name='commentid', primary_key=True, max_length=64)
+    is_hidden = BooleanField(column_name='ishidden', constraints=[SQL("DEFAULT 0")])
     claim_id = CharField(max_length=40, column_name='LbryClaimId')
     parent = ForeignKeyField(
         column_name='ParentId',
diff --git a/src/misc.py b/src/misc.py
index e9d5be8..7c3d4bf 100644
--- a/src/misc.py
+++ b/src/misc.py
@@ -16,7 +16,7 @@ async def get_claim_from_id(app, claim_id, **kwargs):
 
 def clean_input_params(kwargs: dict):
     for k, v in kwargs.items():
-        if type(v) is str and k is not 'comment':
+        if type(v) is str and k != 'comment':
             kwargs[k] = v.strip()
             if k in ID_LIST:
                 kwargs[k] = v.lower()
diff --git a/src/server/app.py b/src/server/app.py
index 389083e..37e89da 100644
--- a/src/server/app.py
+++ b/src/server/app.py
@@ -28,6 +28,7 @@ def setup_database(app):
             host=config[mode]['host'],
             password=config[mode]['password'],
             port=config[mode]['port'],
+            charset=config[mode]['charset'],
         )
     elif config[mode]['database'] == 'sqlite':
         app['db'] = SqliteDatabase(

From c7e8d274f7555786100a576c1559bd42dac4af8f Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Mon, 6 Apr 2020 19:26:22 -0400
Subject: [PATCH 39/46] update travis

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index fe81b23..fe9a4de 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,6 +30,5 @@ jobs:
       name: "Unit Tests"
       install:
         - pip install -e .
-        - mkdir database
       script:
         - python -m unittest

From 7b7e6c66acb0dd18b5bf17294e0a3736e497fe17 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 7 Apr 2020 15:04:50 -0400
Subject: [PATCH 40/46] `VARCHAR` -> `CHAR` for all ID fields, constraints
 moved out of table definition

---
 database/default_after.sql | 39 ++++++++++++++++++++++----------------
 src/database/models.py     | 20 ++++++++-----------
 2 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/database/default_after.sql b/database/default_after.sql
index 1ee19f4..e6a606b 100644
--- a/database/default_after.sql
+++ b/database/default_after.sql
@@ -6,8 +6,7 @@ ALTER DATABASE `social`
 DROP TABLE IF EXISTS `CHANNEL`;
 CREATE TABLE `CHANNEL` (
         `claimid` VARCHAR(40)  NOT NULL,
-        -- i cant tell if max name length is 255 or 256
-        `name`  VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+        `name`  CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
         CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`)
     )
 CHARACTER SET utf8mb4
@@ -15,34 +14,42 @@ COLLATE utf8mb4_unicode_ci;
 
 DROP TABLE IF EXISTS `COMMENT`;
 CREATE TABLE `COMMENT` (
-        `commentid`   VARCHAR(64)   NOT NULL,
-        `lbryclaimid` VARCHAR(40)   NOT NULL,
-        `channelid`   VARCHAR(40)              DEFAULT NULL,
-        `body`        VARCHAR(5000)
+        -- should be changed to CHAR(64)
+        `commentid`   CHAR(64)   NOT NULL,
+        -- should be changed to CHAR(40)
+        `lbryclaimid` CHAR(40)   NOT NULL,
+        -- can be null, so idk if this should be char(40)
+        `channelid`   CHAR(40)              DEFAULT NULL,
+        `body`        TEXT
             CHARACTER SET utf8mb4
             COLLATE utf8mb4_unicode_ci
             NOT NULL,
-        `parentid`    VARCHAR(64)               DEFAULT NULL,
-        `signature`   VARCHAR(128)              DEFAULT NULL,
+        `parentid`    CHAR(64)               DEFAULT NULL,
+        `signature`   CHAR(128)              DEFAULT NULL,
+        -- 22 chars long is prolly enough
         `signingts`   VARCHAR(22)               DEFAULT NULL,
 
         `timestamp`   INTEGER NOT NULL,
         -- there's no way that the timestamp will ever reach 22 characters
         `ishidden`    BOOLEAN                    DEFAULT FALSE,
-        CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`),
-        CONSTRAINT `comment_signature_sk` UNIQUE (`signature`),
-        CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`)
-            ON DELETE CASCADE ON UPDATE CASCADE,
-        CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`)
-            ON UPDATE CASCADE ON DELETE CASCADE, -- setting null implies comment is top level
-        CONSTRAINT `channel_signature`
-            CHECK ( `signature` IS NOT NULL AND `signingts` IS NOT NULL)
+        CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`)
+         -- setting null implies comment is top level
     )
 CHARACTER SET utf8mb4
 COLLATE utf8mb4_unicode_ci;
 
+
+ALTER TABLE COMMENT
+    ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`)
+        ON DELETE CASCADE ON UPDATE CASCADE,
+    ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`)
+            ON UPDATE CASCADE ON DELETE CASCADE
+;
+
 CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`);
 CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`);
 
 
+ALTER TABLE COMMENT ADD CONSTRAINT UNIQUE (`signature`, `channelid`);
+
 
diff --git a/src/database/models.py b/src/database/models.py
index 9e72d33..b35f122 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -13,19 +13,15 @@ from src.misc import clean
 
 
 class Channel(Model):
-    claim_id = CharField(column_name='claimid', primary_key=True, max_length=40)
-    name = CharField(column_name='name', max_length=256)
+    claim_id = FixedCharField(column_name='claimid', primary_key=True, max_length=40)
+    name = CharField(column_name='name', max_length=255)
 
     class Meta:
         table_name = 'CHANNEL'
 
 
 class Comment(Model):
-    comment = CharField(
-        column_name='body',
-        max_length=5000,
-
-    )
+    comment = TextField(column_name='body')
     channel = ForeignKeyField(
         backref='comments',
         column_name='channelid',
@@ -33,9 +29,9 @@ class Comment(Model):
         model=Channel,
         null=True
     )
-    comment_id = CharField(column_name='commentid', primary_key=True, max_length=64)
+    comment_id = FixedCharField(column_name='commentid', primary_key=True, max_length=64)
     is_hidden = BooleanField(column_name='ishidden', constraints=[SQL("DEFAULT 0")])
-    claim_id = CharField(max_length=40, column_name='LbryClaimId')
+    claim_id = FixedCharField(max_length=40, column_name='lbryclaimid')
     parent = ForeignKeyField(
         column_name='ParentId',
         field='comment_id',
@@ -43,9 +39,9 @@ class Comment(Model):
         null=True,
         backref='replies'
     )
-    signature = CharField(max_length=128, column_name='Signature', null=True, unique=True)
-    signing_ts = TextField(column_name='SigningTs', null=True)
-    timestamp = IntegerField(column_name='Timestamp')
+    signature = FixedCharField(max_length=128, column_name='signature', null=True, unique=True)
+    signing_ts = TextField(column_name='signingts', null=True)
+    timestamp = IntegerField(column_name='timestamp')
 
     class Meta:
         table_name = 'COMMENT'

From 0817b700830316429c5ac53fca47c1ef635f98d8 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 7 Apr 2020 16:08:29 -0400
Subject: [PATCH 41/46] Update README.md

---
 README.md | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 471f552..cdeef1d 100644
--- a/README.md
+++ b/README.md
@@ -3,34 +3,36 @@
 [![Build Status](https://travis-ci.com/lbryio/comment-server.svg?branch=master)](https://travis-ci.com/lbryio/comment-server)
 [![Maintainability](https://api.codeclimate.com/v1/badges/22f420b8b5f2373fd885/maintainability)](https://codeclimate.com/github/lbryio/comment-server/maintainability)  
 
-This is the code for the LBRY Comment Server. 
-Fork it, run it, set it on fire. Up to you.
-
 
 ## Before Installing
 
-Comment Deletion requires having the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) 
+Install the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) 
 in order to validate & properly delete comments. 
 
 
- 
- 
+
 ## Installation
 
 #### Installing the server:
 ```bash
 
-$ git clone https://github.com/osilkin98/comment-server
+$ git clone https://github.com/lbryio/comment-server
 $ cd comment-server
 
 # create a virtual environment
-$ virtualenv --python=python3 venv
+$ virtualenv --python=python3.8 venv
 
 # Enter the virtual environment
 $ source venv/bin/activate
 
-# install the Server as a Executable Target
-(venv) $ python setup.py develop
+# Install required dependencies
+(venv) $ pip install -e .
+
+# Run the server
+(venv) $ python src/main.py \
+          --port=5921 \          # use a different port besides the default
+          --config=conf.yml \   # provide a custom config file
+           & \  # detach and run the service in the background 
 ```
 
 ### Installing the systemd Service Monitor
@@ -70,16 +72,11 @@ To Test the database, simply run:
 
 There are basic tests to run against the server, though they require 
 that there is a server instance running, though the database
- chosen may have to be edited in `config/conf.json`.
+ chosen may have to be edited in `config/conf.yml`.
 
 Additionally there are HTTP requests that can be send with whatever 
 software you choose to test the integrity of the comment server.
 
-## Schema
-
-
-![schema](schema.png)
-
 
 ## Contributing
 Contributions are welcome, verbosity is encouraged. Please be considerate

From 3b91279cc73858d73f256df43994d2f992b3e7ab Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 7 Apr 2020 16:16:15 -0400
Subject: [PATCH 42/46] Moves schema from SQLite3 to MySQL

---
 database/default_after.sql    |  55 ----------------
 src/database/comments_ddl.sql | 114 +++++++++++++---------------------
 src/database/schema.py        |  76 -----------------------
 3 files changed, 44 insertions(+), 201 deletions(-)
 delete mode 100644 database/default_after.sql
 delete mode 100644 src/database/schema.py

diff --git a/database/default_after.sql b/database/default_after.sql
deleted file mode 100644
index e6a606b..0000000
--- a/database/default_after.sql
+++ /dev/null
@@ -1,55 +0,0 @@
-USE `social`;
-ALTER DATABASE `social`
-    DEFAULT CHARACTER SET utf8mb4
-    DEFAULT COLLATE utf8mb4_unicode_ci;
-
-DROP TABLE IF EXISTS `CHANNEL`;
-CREATE TABLE `CHANNEL` (
-        `claimid` VARCHAR(40)  NOT NULL,
-        `name`  CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
-        CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`)
-    )
-CHARACTER SET utf8mb4
-COLLATE utf8mb4_unicode_ci;
-
-DROP TABLE IF EXISTS `COMMENT`;
-CREATE TABLE `COMMENT` (
-        -- should be changed to CHAR(64)
-        `commentid`   CHAR(64)   NOT NULL,
-        -- should be changed to CHAR(40)
-        `lbryclaimid` CHAR(40)   NOT NULL,
-        -- can be null, so idk if this should be char(40)
-        `channelid`   CHAR(40)              DEFAULT NULL,
-        `body`        TEXT
-            CHARACTER SET utf8mb4
-            COLLATE utf8mb4_unicode_ci
-            NOT NULL,
-        `parentid`    CHAR(64)               DEFAULT NULL,
-        `signature`   CHAR(128)              DEFAULT NULL,
-        -- 22 chars long is prolly enough
-        `signingts`   VARCHAR(22)               DEFAULT NULL,
-
-        `timestamp`   INTEGER NOT NULL,
-        -- there's no way that the timestamp will ever reach 22 characters
-        `ishidden`    BOOLEAN                    DEFAULT FALSE,
-        CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`)
-         -- setting null implies comment is top level
-    )
-CHARACTER SET utf8mb4
-COLLATE utf8mb4_unicode_ci;
-
-
-ALTER TABLE COMMENT
-    ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`)
-        ON DELETE CASCADE ON UPDATE CASCADE,
-    ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`)
-            ON UPDATE CASCADE ON DELETE CASCADE
-;
-
-CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`);
-CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`);
-
-
-ALTER TABLE COMMENT ADD CONSTRAINT UNIQUE (`signature`, `channelid`);
-
-
diff --git a/src/database/comments_ddl.sql b/src/database/comments_ddl.sql
index 2dfc19d..f488406 100644
--- a/src/database/comments_ddl.sql
+++ b/src/database/comments_ddl.sql
@@ -1,76 +1,50 @@
-PRAGMA FOREIGN_KEYS = ON;
+USE `social`;
+ALTER DATABASE `social`
+    DEFAULT CHARACTER SET utf8mb4
+    DEFAULT COLLATE utf8mb4_unicode_ci;
 
--- Although I know this file is unnecessary, I like keeping it around.
+DROP TABLE IF EXISTS `CHANNEL`;
+CREATE TABLE `CHANNEL` (
+        `claimid` VARCHAR(40)  NOT NULL,
+        `name`  CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+        CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`)
+    )
+CHARACTER SET utf8mb4
+COLLATE utf8mb4_unicode_ci;
 
--- I'm not gonna remove it.
+DROP TABLE IF EXISTS `COMMENT`;
+CREATE TABLE `COMMENT` (
+        -- should be changed to CHAR(64)
+        `commentid`   CHAR(64)   NOT NULL,
+        -- should be changed to CHAR(40)
+        `lbryclaimid` CHAR(40)   NOT NULL,
+        -- can be null, so idk if this should be char(40)
+        `channelid`   CHAR(40)              DEFAULT NULL,
+        `body`        TEXT
+            CHARACTER SET utf8mb4
+            COLLATE utf8mb4_unicode_ci
+            NOT NULL,
+        `parentid`    CHAR(64)               DEFAULT NULL,
+        `signature`   CHAR(128)              DEFAULT NULL,
+        -- 22 chars long is prolly enough
+        `signingts`   VARCHAR(22)               DEFAULT NULL,
 
--- tables
-CREATE TABLE IF NOT EXISTS COMMENT
-(
-    CommentId   TEXT    NOT NULL,
-    LbryClaimId TEXT    NOT NULL,
-    ChannelId   TEXT             DEFAULT NULL,
-    Body        TEXT    NOT NULL,
-    ParentId    TEXT             DEFAULT NULL,
-    Signature   TEXT             DEFAULT NULL,
-    Timestamp   INTEGER NOT NULL,
-    SigningTs   TEXT             DEFAULT NULL,
-    IsHidden    BOOLEAN NOT NULL DEFAULT FALSE,
-    CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
-    CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
-    CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
-        ON DELETE NO ACTION ON UPDATE NO ACTION,
-    CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId)
-        ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
-);
-
--- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE);
--- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL;
-
--- DROP TABLE IF EXISTS CHANNEL;
-CREATE TABLE IF NOT EXISTS CHANNEL
-(
-    ClaimId TEXT NOT NULL,
-    Name    TEXT NOT NULL,
-    CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId)
-        ON CONFLICT IGNORE
-);
+        `timestamp`   INTEGER NOT NULL,
+        -- there's no way that the timestamp will ever reach 22 characters
+        `ishidden`    BOOLEAN                    DEFAULT FALSE,
+        CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`)
+         -- setting null implies comment is top level
+    )
+CHARACTER SET utf8mb4
+COLLATE utf8mb4_unicode_ci;
 
 
--- indexes
--- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX;
--- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
+ALTER TABLE COMMENT
+    ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`)
+        ON DELETE CASCADE ON UPDATE CASCADE,
+    ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`)
+            ON UPDATE CASCADE ON DELETE CASCADE
+;
 
--- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
-
--- VIEWS
-CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS
-SELECT C.CommentId                                     AS comment_id,
-       C.Body                                          AS comment,
-       C.LbryClaimId                                   AS claim_id,
-       C.Timestamp                                     AS timestamp,
-       CHAN.Name                                       AS channel_name,
-       CHAN.ClaimId                                    AS channel_id,
-       ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
-       C.Signature                                     AS signature,
-       C.SigningTs                                     AS signing_ts,
-       C.ParentId                                      AS parent_id,
-       C.IsHidden                                      AS is_hidden
-FROM COMMENT AS C
-         LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
-ORDER BY C.Timestamp DESC;
-
-
-DROP VIEW IF EXISTS COMMENT_REPLIES;
-CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS
-SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body
-FROM COMMENT AS OG
-         JOIN COMMENT AS PARENT
-              ON OG.ParentId = PARENT.CommentId
-         JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId
-         JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId
-ORDER BY OG.Timestamp;
-
--- this is the default channel for anyone who wants to publish anonymously
--- INSERT INTO CHANNEL
--- VALUES ('9cb713f01bf247a0e03170b5ed00d5161340c486', '@Anonymous');
+CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`);
+CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`);
diff --git a/src/database/schema.py b/src/database/schema.py
deleted file mode 100644
index c75681b..0000000
--- a/src/database/schema.py
+++ /dev/null
@@ -1,76 +0,0 @@
-PRAGMAS = """
-    PRAGMA FOREIGN_KEYS = ON;
-"""
-
-CREATE_COMMENT_TABLE = """
-    CREATE TABLE IF NOT EXISTS COMMENT (
-        CommentId   TEXT    NOT NULL,
-        LbryClaimId TEXT    NOT NULL,
-        ChannelId   TEXT                DEFAULT NULL,
-        Body        TEXT    NOT NULL,
-        ParentId    TEXT                DEFAULT NULL,
-        Signature   TEXT                DEFAULT NULL,
-        Timestamp   INTEGER NOT NULL,
-        SigningTs   TEXT                DEFAULT NULL,
-        IsHidden    BOOLEAN NOT NULL    DEFAULT 0,
-        CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
-        CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
-        CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
-            ON DELETE NO ACTION ON UPDATE NO ACTION,
-        CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId)
-            ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
-    );
-"""
-
-CREATE_COMMENT_INDEXES = """
-    CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
-    CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
-"""
-
-CREATE_CHANNEL_TABLE = """
-    CREATE TABLE IF NOT EXISTS CHANNEL (
-        ClaimId TEXT NOT NULL,
-        Name    TEXT NOT NULL,
-        CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId)
-            ON CONFLICT IGNORE
-    );
-"""
-
-CREATE_COMMENTS_ON_CLAIMS_VIEW = """
-    CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT 
-        C.CommentId AS comment_id,
-        C.Body AS comment,
-        C.LbryClaimId AS claim_id,
-        C.Timestamp AS timestamp,
-        CHAN.Name AS channel_name,
-        CHAN.ClaimId AS channel_id,
-        ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
-        C.Signature AS signature,
-        C.SigningTs AS signing_ts,
-        C.ParentId AS parent_id,
-        C.IsHidden AS is_hidden
-    FROM COMMENT AS C
-             LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
-    ORDER BY C.Timestamp DESC;
-"""
-
-# not being used right now but should be kept around when Tom finally asks for replies
-CREATE_COMMENT_REPLIES_VIEW = """
-CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS
-SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body
-FROM COMMENT AS OG
-         JOIN COMMENT AS PARENT
-              ON OG.ParentId = PARENT.CommentId
-         JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId
-         JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId
-ORDER BY OG.Timestamp;
-"""
-
-CREATE_TABLES_QUERY = (
-        PRAGMAS +
-        CREATE_COMMENT_TABLE +
-        CREATE_COMMENT_INDEXES +
-        CREATE_CHANNEL_TABLE +
-        CREATE_COMMENTS_ON_CLAIMS_VIEW +
-        CREATE_COMMENT_REPLIES_VIEW
-)

From b4377b2f54ae06403cb978f5084e0b271de4be6c Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Tue, 7 Apr 2020 16:16:34 -0400
Subject: [PATCH 43/46] Cleans up & adds todos for future

---
 scripts/stress_test.py      | 84 -------------------------------------
 scripts/valid_signatures.py |  3 +-
 src/database/writes.py      |  1 +
 src/main.py                 |  1 -
 4 files changed, 3 insertions(+), 86 deletions(-)
 delete mode 100644 scripts/stress_test.py

diff --git a/scripts/stress_test.py b/scripts/stress_test.py
deleted file mode 100644
index 6022396..0000000
--- a/scripts/stress_test.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import sqlite3
-import time
-
-import faker
-from faker.providers import misc
-
-fake = faker.Faker()
-fake.add_provider(misc)
-
-
-if __name__ == '__main__':
-    song_time = """One, two, three!
-My baby don't mess around
-'Cause she loves me so
-This I know fo sho!
-But does she really wanna
-But can't stand to see me walk out tha door
-Don't try to fight the feeling
-Because the thought alone is killin' me right now
-Thank God for Mom and Dad
-For sticking to together
-Like we don't know how
-Hey ya! Hey ya!
-Hey ya! Hey ya!
-Hey ya! Hey ya!
-Hey ya! Hey ya!
-You think you've got it
-Oh, you think you've got it
-But got it just don't get it when there's nothin' at all
-We get together
-Oh, we get together
-But separate's always better when there's feelings involved
-Know what they say -its
-Nothing lasts forever!
-Then what makes it, then what makes it
-Then what makes it, then what makes it
-Then what makes love the exception?
-So why, oh, why, oh
-Why, oh, why, oh, why, oh
-Are we still in denial when we know we're not happy here
-Hey ya! (y'all don't want to here me, ya just want to dance) Hey ya!
-Don't want to meet your daddy (oh ohh), just want you in my caddy (oh ohh)
-Hey ya! (oh, oh!) Hey ya! (oh, oh!)
-Don't want to meet your momma, just want to make you cum-a (oh, oh!)
-I'm (oh, oh) I'm (oh, oh) I'm just being honest! (oh, oh)
-I'm just being honest!
-Hey! alright now! alright now, fellas!
-Yea?
-Now, what cooler than being cool?
-Ice cold!
-I can't hear ya! I say what's, what's cooler than being cool?
-Ice cold!
-Alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright!
-Okay, now ladies!
-Yea?
-Now we gonna break this thang down for just a few seconds
-Now don't have me break this thang down for nothin'
-I want to see you on your badest behavior!
-Lend me some sugar, I am your neighbor!
-Ah! Here we go now,
-Shake it, shake it, shake it, shake it, shake it
-Shake it, shake it, shake it, shake it
-Shake it like a Polaroid picture! Hey ya!
-Shake it, shake it, shake it, shake it, shake it
-Shake it, shake it, shake it, suga!
-Shake it like a Polaroid picture!
-Now all the Beyonce's, and Lucy Lu's, and baby dolls
-Get on tha floor get on tha floor!
-Shake it like a Polaroid picture!
-Oh, you! oh, you!
-Hey ya!(oh, oh) Hey ya!(oh, oh)
-Hey ya!(oh, oh) Hey ya!(oh, oh)
-Hey ya!(oh, oh) Hey ya!(oh, oh)"""
-
-    song = song_time.split('\n')
-    claim_id = '2aa106927b733e2602ffb565efaccc78c2ed89df'
-    run_len = [(fake.sha256(), song_time, claim_id, str(int(time.time()))) for k in range(5000)]
-
-    conn = sqlite3.connect('database/default_test.db')
-    with conn:
-        curs = conn.executemany("""
-        INSERT INTO COMMENT(CommentId, Body, LbryClaimId, Timestamp) VALUES (?, ?, ?, ?)
-        """, run_len)
-        print(f'rows changed: {curs.rowcount}')
diff --git a/scripts/valid_signatures.py b/scripts/valid_signatures.py
index 19fc878..4b7c641 100644
--- a/scripts/valid_signatures.py
+++ b/scripts/valid_signatures.py
@@ -2,11 +2,12 @@ import binascii
 import logging
 import hashlib
 import json
+# todo: remove sqlite3 as a dependency
 import sqlite3
 import asyncio
 import aiohttp
 
-from server.validation import is_signature_valid, get_encoded_signature
+from src.server.validation import is_signature_valid, get_encoded_signature
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/database/writes.py b/src/database/writes.py
index 61caa78..b86e31b 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -1,3 +1,4 @@
+# TODO: scrap notification routines from these files & supply them in handles
 import logging
 import sqlite3
 from asyncio import coroutine
diff --git a/src/main.py b/src/main.py
index 817dac4..d4b5a53 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1,5 +1,4 @@
 import argparse
-import json
 import yaml
 import logging
 import logging.config

From 3cce89cbacc315a6e896e46e0a98eb3ea2b23510 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Thu, 9 Apr 2020 16:55:07 -0400
Subject: [PATCH 44/46] serializes error messages into json format

---
 src/server/errors.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/server/errors.py b/src/server/errors.py
index 273e4bd..644ce9e 100644
--- a/src/server/errors.py
+++ b/src/server/errors.py
@@ -1,3 +1,5 @@
+import json
+
 import logging
 import aiohttp
 
@@ -32,8 +34,11 @@ def make_error(error, exc=None) -> dict:
 async def report_error(app, exc, body: dict):
     try:
         if 'slack_webhook' in app['config']:
+            body_dump = json.dumps(body, indent=4)
+            exec_name = type(exc).__name__
+            exec_body = str(exc)
             message = {
-                "text": f"Got `{type(exc).__name__}`: `\n{str(exc)}`\n```{body}```"
+                "text": f"Got `{exec_name}`: `\n{exec_body}`\n```{body_dump}```"
             }
             async with aiohttp.ClientSession() as sesh:
                 async with sesh.post(app['config']['slack_webhook'], json=message) as resp:

From 7c26b809713dea812ce0df0b75a8f67872903fa0 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Tue, 21 Jul 2020 02:35:33 -0400
Subject: [PATCH 45/46] Fix notifications

---
 src/server/handles.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/server/handles.py b/src/server/handles.py
index 1c56663..a7fe43b 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -7,6 +7,7 @@ from aiohttp import web
 from aiojobs.aiohttp import atomic
 from peewee import DoesNotExist
 
+from src.server.external import send_notification
 from src.server.validation import validate_signature_from_claim
 from src.misc import clean_input_params, get_claim_from_id
 from src.server.errors import make_error, report_error
@@ -116,7 +117,7 @@ async def handle_abandon_comment(
     else:
         if not validate_signature_from_claim(channel, signature, signing_ts, comment_id):
             raise ValueError('Abandon signature could not be validated')
-
+    await app['webhooks'].spawn(send_notification(app, 'DELETE', comment))
     with app['db'].atomic():
         return {
             'abandoned': delete_comment(comment_id)
@@ -184,15 +185,17 @@ async def handle_edit_comment(app, comment: str = None, comment_id: str = None,
     with app['db'].atomic():
         if not edit_comment(comment_id, comment, signature, signing_ts):
             raise ValueError('Comment could not be edited')
-        return get_comment(comment_id)
+        updated_comment = get_comment(comment_id)
+        await app['webhooks'].spawn(send_notification(app, 'UPDATE', updated_comment))
+        return updated_comment
 
 
 # TODO: retrieve stake amounts for each channel & store in db
-def handle_create_comment(app, comment: str = None, claim_id: str = None,
+async def handle_create_comment(app, comment: str = None, claim_id: str = None,
                           parent_id: str = None, channel_id: str = None, channel_name: str = None,
                           signature: str = None, signing_ts: str = None) -> dict:
     with app['db'].atomic():
-        return create_comment(
+        comment = create_comment(
             comment=comment,
             claim_id=claim_id,
             parent_id=parent_id,
@@ -201,6 +204,8 @@ def handle_create_comment(app, comment: str = None, claim_id: str = None,
             signature=signature,
             signing_ts=signing_ts
         )
+        await app['webhooks'].spawn(send_notification(app, 'CREATE', comment))
+        return comment
 
 
 METHODS = {

From c2230cdefb05c904051b0567b08d89427df3bb1a Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Fri, 24 Jul 2020 15:43:34 -0400
Subject: [PATCH 46/46] Send parent id and comment text to internal-apis for
 notifications

---
 src/server/external.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/server/external.py b/src/server/external.py
index aa0e7ee..25bd634 100644
--- a/src/server/external.py
+++ b/src/server/external.py
@@ -37,6 +37,10 @@ def create_notification_batch(action: str, comments: List[dict]) -> List[dict]:
         }
         if comment.get('channel_id'):
             event['channel_id'] = comment['channel_id']
+        if comment.get('parent_id'):
+            event['parent_id'] = comment['parent_id']
+        if comment.get('comment'):
+            event['comment'] = comment['comment']
         events.append(event)
     return events