Send parent id and comment text to internal-apis for notifications

Fix notifications

.travis.yml

@@ -30,6 +30,5 @@ jobs:
       name: "Unit Tests"
       install:
         - pip install -e .
-        - mkdir database
       script:
         - python -m unittest

README.md

@@ -3,34 +3,36 @@
 [![Build Status](https://travis-ci.com/lbryio/comment-server.svg?branch=master)](https://travis-ci.com/lbryio/comment-server)
 [![Maintainability](https://api.codeclimate.com/v1/badges/22f420b8b5f2373fd885/maintainability)](https://codeclimate.com/github/lbryio/comment-server/maintainability)  
 
-This is the code for the LBRY Comment Server. 
-Fork it, run it, set it on fire. Up to you.
-
 
 ## Before Installing
 
-Comment Deletion requires having the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) 
+Install the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) 
 in order to validate & properly delete comments. 
 
 
- 
+
- 
 ## Installation
 
 #### Installing the server:
 ```bash
 
-$ git clone https://github.com/osilkin98/comment-server
+$ git clone https://github.com/lbryio/comment-server
 $ cd comment-server
 
 # create a virtual environment
-$ virtualenv --python=python3 venv
+$ virtualenv --python=python3.8 venv
 
 # Enter the virtual environment
 $ source venv/bin/activate
 
-# install the Server as a Executable Target
+# Install required dependencies
-(venv) $ python setup.py develop
+(venv) $ pip install -e .
+
+# Run the server
+(venv) $ python src/main.py \
+          --port=5921 \          # use a different port besides the default
+          --config=conf.yml \   # provide a custom config file
+           & \  # detach and run the service in the background 
 ```
 
 ### Installing the systemd Service Monitor
@@ -70,16 +72,11 @@ To Test the database, simply run:
 
 There are basic tests to run against the server, though they require 
 that there is a server instance running, though the database
- chosen may have to be edited in `config/conf.json`.
+ chosen may have to be edited in `config/conf.yml`.
 
 Additionally there are HTTP requests that can be send with whatever 
 software you choose to test the integrity of the comment server.
 
-## Schema
-
-
-![schema](schema.png)
-
 
 ## Contributing
 Contributions are welcome, verbosity is encouraged. Please be considerate

config/conf.yml

@@ -12,8 +12,9 @@ testing:
 
 # actual database should be running MySQL
 production:
+  charset: utf8mb4
   database: mysql
-  name: lbry
+  name: social
   user: lbry
   password: lbry
   host: localhost

docker-compose.yml

@@ -6,11 +6,12 @@ services:
   mysql:
     image: mysql/mysql-server:5.7.27
     restart: "no"
+    command: --character_set_server=utf8mb4 --max_allowed_packet=1073741824
     ports:
       - "3306:3306"
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=true
-      - MYSQL_DATABASE=lbry
+      - MYSQL_DATABASE=social
       - MYSQL_USER=lbry
       - MYSQL_PASSWORD=lbry
       - MYSQL_LOG_CONSOLE=true

scripts/stress_test.py

@@ -1,84 +0,0 @@
-import sqlite3
-import time
-
-import faker
-from faker.providers import misc
-
-fake = faker.Faker()
-fake.add_provider(misc)
-
-
-if __name__ == '__main__':
-    song_time = """One, two, three!
-My baby don't mess around
-'Cause she loves me so
-This I know fo sho!
-But does she really wanna
-But can't stand to see me walk out tha door
-Don't try to fight the feeling
-Because the thought alone is killin' me right now
-Thank God for Mom and Dad
-For sticking to together
-Like we don't know how
-Hey ya! Hey ya!
-Hey ya! Hey ya!
-Hey ya! Hey ya!
-Hey ya! Hey ya!
-You think you've got it
-Oh, you think you've got it
-But got it just don't get it when there's nothin' at all
-We get together
-Oh, we get together
-But separate's always better when there's feelings involved
-Know what they say -its
-Nothing lasts forever!
-Then what makes it, then what makes it
-Then what makes it, then what makes it
-Then what makes love the exception?
-So why, oh, why, oh
-Why, oh, why, oh, why, oh
-Are we still in denial when we know we're not happy here
-Hey ya! (y'all don't want to here me, ya just want to dance) Hey ya!
-Don't want to meet your daddy (oh ohh), just want you in my caddy (oh ohh)
-Hey ya! (oh, oh!) Hey ya! (oh, oh!)
-Don't want to meet your momma, just want to make you cum-a (oh, oh!)
-I'm (oh, oh) I'm (oh, oh) I'm just being honest! (oh, oh)
-I'm just being honest!
-Hey! alright now! alright now, fellas!
-Yea?
-Now, what cooler than being cool?
-Ice cold!
-I can't hear ya! I say what's, what's cooler than being cool?
-Ice cold!
-Alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright!
-Okay, now ladies!
-Yea?
-Now we gonna break this thang down for just a few seconds
-Now don't have me break this thang down for nothin'
-I want to see you on your badest behavior!
-Lend me some sugar, I am your neighbor!
-Ah! Here we go now,
-Shake it, shake it, shake it, shake it, shake it
-Shake it, shake it, shake it, shake it
-Shake it like a Polaroid picture! Hey ya!
-Shake it, shake it, shake it, shake it, shake it
-Shake it, shake it, shake it, suga!
-Shake it like a Polaroid picture!
-Now all the Beyonce's, and Lucy Lu's, and baby dolls
-Get on tha floor get on tha floor!
-Shake it like a Polaroid picture!
-Oh, you! oh, you!
-Hey ya!(oh, oh) Hey ya!(oh, oh)
-Hey ya!(oh, oh) Hey ya!(oh, oh)
-Hey ya!(oh, oh) Hey ya!(oh, oh)"""
-
-    song = song_time.split('\n')
-    claim_id = '2aa106927b733e2602ffb565efaccc78c2ed89df'
-    run_len = [(fake.sha256(), song_time, claim_id, str(int(time.time()))) for k in range(5000)]
-
-    conn = sqlite3.connect('database/default_test.db')
-    with conn:
-        curs = conn.executemany("""
-        INSERT INTO COMMENT(CommentId, Body, LbryClaimId, Timestamp) VALUES (?, ?, ?, ?)
-        """, run_len)
-        print(f'rows changed: {curs.rowcount}')

scripts/valid_signatures.py

@@ -2,11 +2,12 @@ import binascii
 import logging
 import hashlib
 import json
+# todo: remove sqlite3 as a dependency
 import sqlite3
 import asyncio
 import aiohttp
 
-from server.validation import is_signature_valid, get_encoded_signature
+from src.server.validation import is_signature_valid, get_encoded_signature
 
 logger = logging.getLogger(__name__)
 

src/database/comments_ddl.sql

@@ -1,76 +1,50 @@
-PRAGMA FOREIGN_KEYS = ON;
+USE `social`;
+ALTER DATABASE `social`
+    DEFAULT CHARACTER SET utf8mb4
+    DEFAULT COLLATE utf8mb4_unicode_ci;
 
--- Although I know this file is unnecessary, I like keeping it around.
+DROP TABLE IF EXISTS `CHANNEL`;
+CREATE TABLE `CHANNEL` (
+        `claimid` VARCHAR(40)  NOT NULL,
+        `name`  CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+        CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`)
+    )
+CHARACTER SET utf8mb4
+COLLATE utf8mb4_unicode_ci;
 
--- I'm not gonna remove it.
+DROP TABLE IF EXISTS `COMMENT`;
+CREATE TABLE `COMMENT` (
+        -- should be changed to CHAR(64)
+        `commentid`   CHAR(64)   NOT NULL,
+        -- should be changed to CHAR(40)
+        `lbryclaimid` CHAR(40)   NOT NULL,
+        -- can be null, so idk if this should be char(40)
+        `channelid`   CHAR(40)              DEFAULT NULL,
+        `body`        TEXT
+            CHARACTER SET utf8mb4
+            COLLATE utf8mb4_unicode_ci
+            NOT NULL,
+        `parentid`    CHAR(64)               DEFAULT NULL,
+        `signature`   CHAR(128)              DEFAULT NULL,
+        -- 22 chars long is prolly enough
+        `signingts`   VARCHAR(22)               DEFAULT NULL,
 
--- tables
+        `timestamp`   INTEGER NOT NULL,
-CREATE TABLE IF NOT EXISTS COMMENT
+        -- there's no way that the timestamp will ever reach 22 characters
-(
+        `ishidden`    BOOLEAN                    DEFAULT FALSE,
-    CommentId   TEXT    NOT NULL,
+        CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`)
-    LbryClaimId TEXT    NOT NULL,
+         -- setting null implies comment is top level
-    ChannelId   TEXT             DEFAULT NULL,
+    )
-    Body        TEXT    NOT NULL,
+CHARACTER SET utf8mb4
-    ParentId    TEXT             DEFAULT NULL,
+COLLATE utf8mb4_unicode_ci;
-    Signature   TEXT             DEFAULT NULL,
-    Timestamp   INTEGER NOT NULL,
-    SigningTs   TEXT             DEFAULT NULL,
-    IsHidden    BOOLEAN NOT NULL DEFAULT FALSE,
-    CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
-    CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
-    CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
-        ON DELETE NO ACTION ON UPDATE NO ACTION,
-    CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId)
-        ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
-);
-
--- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE);
--- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL;
-
--- DROP TABLE IF EXISTS CHANNEL;
-CREATE TABLE IF NOT EXISTS CHANNEL
-(
-    ClaimId TEXT NOT NULL,
-    Name    TEXT NOT NULL,
-    CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId)
-        ON CONFLICT IGNORE
-);
 
 
--- indexes
+ALTER TABLE COMMENT
--- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX;
+    ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`)
--- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
+        ON DELETE CASCADE ON UPDATE CASCADE,
+    ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`)
+            ON UPDATE CASCADE ON DELETE CASCADE
+;
 
--- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
+CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`);
-
+CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`);
--- VIEWS
-CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS
-SELECT C.CommentId                                     AS comment_id,
-       C.Body                                          AS comment,
-       C.LbryClaimId                                   AS claim_id,
-       C.Timestamp                                     AS timestamp,
-       CHAN.Name                                       AS channel_name,
-       CHAN.ClaimId                                    AS channel_id,
-       ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
-       C.Signature                                     AS signature,
-       C.SigningTs                                     AS signing_ts,
-       C.ParentId                                      AS parent_id,
-       C.IsHidden                                      AS is_hidden
-FROM COMMENT AS C
-         LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
-ORDER BY C.Timestamp DESC;
-
-
-DROP VIEW IF EXISTS COMMENT_REPLIES;
-CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS
-SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body
-FROM COMMENT AS OG
-         JOIN COMMENT AS PARENT
-              ON OG.ParentId = PARENT.CommentId
-         JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId
-         JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId
-ORDER BY OG.Timestamp;
-
--- this is the default channel for anyone who wants to publish anonymously
--- INSERT INTO CHANNEL
--- VALUES ('9cb713f01bf247a0e03170b5ed00d5161340c486', '@Anonymous');

src/database/models.py

@@ -13,25 +13,25 @@ from src.misc import clean
 
 
 class Channel(Model):
-    claim_id = CharField(column_name='ClaimId', primary_key=True, max_length=40)
+    claim_id = FixedCharField(column_name='claimid', primary_key=True, max_length=40)
-    name = CharField(column_name='Name', max_length=256)
+    name = CharField(column_name='name', max_length=255)
 
     class Meta:
         table_name = 'CHANNEL'
 
 
 class Comment(Model):
-    comment = CharField(column_name='Body', max_length=2000)
+    comment = TextField(column_name='body')
     channel = ForeignKeyField(
         backref='comments',
-        column_name='ChannelId',
+        column_name='channelid',
         field='claim_id',
         model=Channel,
         null=True
     )
-    comment_id = CharField(column_name='CommentId', primary_key=True, max_length=64)
+    comment_id = FixedCharField(column_name='commentid', primary_key=True, max_length=64)
-    is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")])
+    is_hidden = BooleanField(column_name='ishidden', constraints=[SQL("DEFAULT 0")])
-    claim_id = CharField(max_length=40, column_name='LbryClaimId')
+    claim_id = FixedCharField(max_length=40, column_name='lbryclaimid')
     parent = ForeignKeyField(
         column_name='ParentId',
         field='comment_id',
@@ -39,9 +39,9 @@ class Comment(Model):
         null=True,
         backref='replies'
     )
-    signature = CharField(max_length=128, column_name='Signature', null=True, unique=True)
+    signature = FixedCharField(max_length=128, column_name='signature', null=True, unique=True)
-    signing_ts = TextField(column_name='SigningTs', null=True)
+    signing_ts = TextField(column_name='signingts', null=True)
-    timestamp = IntegerField(column_name='Timestamp')
+    timestamp = IntegerField(column_name='timestamp')
 
     class Meta:
         table_name = 'COMMENT'

src/database/schema.py

@@ -1,76 +0,0 @@
-PRAGMAS = """
-    PRAGMA FOREIGN_KEYS = ON;
-"""
-
-CREATE_COMMENT_TABLE = """
-    CREATE TABLE IF NOT EXISTS COMMENT (
-        CommentId   TEXT    NOT NULL,
-        LbryClaimId TEXT    NOT NULL,
-        ChannelId   TEXT                DEFAULT NULL,
-        Body        TEXT    NOT NULL,
-        ParentId    TEXT                DEFAULT NULL,
-        Signature   TEXT                DEFAULT NULL,
-        Timestamp   INTEGER NOT NULL,
-        SigningTs   TEXT                DEFAULT NULL,
-        IsHidden    BOOLEAN NOT NULL    DEFAULT 0,
-        CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
-        CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
-        CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
-            ON DELETE NO ACTION ON UPDATE NO ACTION,
-        CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId)
-            ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
-    );
-"""
-
-CREATE_COMMENT_INDEXES = """
-    CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
-    CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
-"""
-
-CREATE_CHANNEL_TABLE = """
-    CREATE TABLE IF NOT EXISTS CHANNEL (
-        ClaimId TEXT NOT NULL,
-        Name    TEXT NOT NULL,
-        CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId)
-            ON CONFLICT IGNORE
-    );
-"""
-
-CREATE_COMMENTS_ON_CLAIMS_VIEW = """
-    CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT 
-        C.CommentId AS comment_id,
-        C.Body AS comment,
-        C.LbryClaimId AS claim_id,
-        C.Timestamp AS timestamp,
-        CHAN.Name AS channel_name,
-        CHAN.ClaimId AS channel_id,
-        ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
-        C.Signature AS signature,
-        C.SigningTs AS signing_ts,
-        C.ParentId AS parent_id,
-        C.IsHidden AS is_hidden
-    FROM COMMENT AS C
-             LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
-    ORDER BY C.Timestamp DESC;
-"""
-
-# not being used right now but should be kept around when Tom finally asks for replies
-CREATE_COMMENT_REPLIES_VIEW = """
-CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS
-SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body
-FROM COMMENT AS OG
-         JOIN COMMENT AS PARENT
-              ON OG.ParentId = PARENT.CommentId
-         JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId
-         JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId
-ORDER BY OG.Timestamp;
-"""
-
-CREATE_TABLES_QUERY = (
-        PRAGMAS +
-        CREATE_COMMENT_TABLE +
-        CREATE_COMMENT_INDEXES +
-        CREATE_CHANNEL_TABLE +
-        CREATE_COMMENTS_ON_CLAIMS_VIEW +
-        CREATE_COMMENT_REPLIES_VIEW
-)

src/database/writes.py

@@ -1,3 +1,4 @@
+# TODO: scrap notification routines from these files & supply them in handles
 import logging
 import sqlite3
 from asyncio import coroutine

src/main.py

@@ -1,5 +1,4 @@
 import argparse
-import json
 import yaml
 import logging
 import logging.config

src/misc.py

@@ -16,7 +16,7 @@ async def get_claim_from_id(app, claim_id, **kwargs):
 
 def clean_input_params(kwargs: dict):
     for k, v in kwargs.items():
-        if type(v) is str and k is not 'comment':
+        if type(v) is str and k != 'comment':
             kwargs[k] = v.strip()
             if k in ID_LIST:
                 kwargs[k] = v.lower()

src/server/app.py

@@ -28,6 +28,7 @@ def setup_database(app):
             host=config[mode]['host'],
             password=config[mode]['password'],
             port=config[mode]['port'],
+            charset=config[mode]['charset'],
         )
     elif config[mode]['database'] == 'sqlite':
         app['db'] = SqliteDatabase(

src/server/errors.py

@@ -1,3 +1,5 @@
+import json
+
 import logging
 import aiohttp
 
@@ -32,8 +34,11 @@ def make_error(error, exc=None) -> dict:
 async def report_error(app, exc, body: dict):
     try:
         if 'slack_webhook' in app['config']:
+            body_dump = json.dumps(body, indent=4)
+            exec_name = type(exc).__name__
+            exec_body = str(exc)
             message = {
-                "text": f"Got `{type(exc).__name__}`: `\n{str(exc)}`\n```{body}```"
+                "text": f"Got `{exec_name}`: `\n{exec_body}`\n```{body_dump}```"
             }
             async with aiohttp.ClientSession() as sesh:
                 async with sesh.post(app['config']['slack_webhook'], json=message) as resp:

src/server/external.py

@@ -37,6 +37,10 @@ def create_notification_batch(action: str, comments: List[dict]) -> List[dict]:
         }
         if comment.get('channel_id'):
             event['channel_id'] = comment['channel_id']
+        if comment.get('parent_id'):
+            event['parent_id'] = comment['parent_id']
+        if comment.get('comment'):
+            event['comment'] = comment['comment']
         events.append(event)
     return events
 

src/server/handles.py

@@ -7,6 +7,7 @@ from aiohttp import web
 from aiojobs.aiohttp import atomic
 from peewee import DoesNotExist
 
+from src.server.external import send_notification
 from src.server.validation import validate_signature_from_claim
 from src.misc import clean_input_params, get_claim_from_id
 from src.server.errors import make_error, report_error
@@ -116,7 +117,7 @@ async def handle_abandon_comment(
     else:
         if not validate_signature_from_claim(channel, signature, signing_ts, comment_id):
             raise ValueError('Abandon signature could not be validated')
-
+    await app['webhooks'].spawn(send_notification(app, 'DELETE', comment))
     with app['db'].atomic():
         return {
             'abandoned': delete_comment(comment_id)
@@ -184,15 +185,17 @@ async def handle_edit_comment(app, comment: str = None, comment_id: str = None,
     with app['db'].atomic():
         if not edit_comment(comment_id, comment, signature, signing_ts):
             raise ValueError('Comment could not be edited')
-        return get_comment(comment_id)
+        updated_comment = get_comment(comment_id)
+        await app['webhooks'].spawn(send_notification(app, 'UPDATE', updated_comment))
+        return updated_comment
 
 
 # TODO: retrieve stake amounts for each channel & store in db
-def handle_create_comment(app, comment: str = None, claim_id: str = None,
+async def handle_create_comment(app, comment: str = None, claim_id: str = None,
                           parent_id: str = None, channel_id: str = None, channel_name: str = None,
                           signature: str = None, signing_ts: str = None) -> dict:
     with app['db'].atomic():
-        return create_comment(
+        comment = create_comment(
             comment=comment,
             claim_id=claim_id,
             parent_id=parent_id,
@@ -201,6 +204,8 @@ def handle_create_comment(app, comment: str = None, claim_id: str = None,
             signature=signature,
             signing_ts=signing_ts
         )
+        await app['webhooks'].spawn(send_notification(app, 'CREATE', comment))
+        return comment
 
 
 METHODS = {