.travis.yml

@@ -30,5 +30,6 @@ jobs:
       name: "Unit Tests"
       install:
         - pip install -e .
+        - mkdir database
       script:
         - python -m unittest

README.md

@@ -3,36 +3,34 @@
 [![Build Status](https://travis-ci.com/lbryio/comment-server.svg?branch=master)](https://travis-ci.com/lbryio/comment-server)
 [![Maintainability](https://api.codeclimate.com/v1/badges/22f420b8b5f2373fd885/maintainability)](https://codeclimate.com/github/lbryio/comment-server/maintainability)  
 
+This is the code for the LBRY Comment Server. 
+Fork it, run it, set it on fire. Up to you.
+
 
 ## Before Installing
 
-Install the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) 
+Comment Deletion requires having the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) 
 in order to validate & properly delete comments. 
 
 
-
+ 
+ 
 ## Installation
 
 #### Installing the server:
 ```bash
 
-$ git clone https://github.com/lbryio/comment-server
+$ git clone https://github.com/osilkin98/comment-server
 $ cd comment-server
 
 # create a virtual environment
-$ virtualenv --python=python3.8 venv
+$ virtualenv --python=python3 venv
 
 # Enter the virtual environment
 $ source venv/bin/activate
 
-# Install required dependencies
+# install the Server as a Executable Target
-(venv) $ pip install -e .
+(venv) $ python setup.py develop
-
-# Run the server
-(venv) $ python src/main.py \
-          --port=5921 \          # use a different port besides the default
-          --config=conf.yml \   # provide a custom config file
-           & \  # detach and run the service in the background 
 ```
 
 ### Installing the systemd Service Monitor
@@ -72,11 +70,16 @@ To Test the database, simply run:
 
 There are basic tests to run against the server, though they require 
 that there is a server instance running, though the database
- chosen may have to be edited in `config/conf.yml`.
+ chosen may have to be edited in `config/conf.json`.
 
 Additionally there are HTTP requests that can be send with whatever 
 software you choose to test the integrity of the comment server.
 
+## Schema
+
+
+![schema](schema.png)
+
 
 ## Contributing
 Contributions are welcome, verbosity is encouraged. Please be considerate

config/conf.yml

@@ -12,9 +12,8 @@ testing:
 
 # actual database should be running MySQL
 production:
-  charset: utf8mb4
   database: mysql
-  name: social
+  name: lbry
   user: lbry
   password: lbry
   host: localhost

docker-compose.yml

@@ -6,12 +6,11 @@ services:
   mysql:
     image: mysql/mysql-server:5.7.27
     restart: "no"
-    command: --character_set_server=utf8mb4 --max_allowed_packet=1073741824
     ports:
       - "3306:3306"
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=true
-      - MYSQL_DATABASE=social
+      - MYSQL_DATABASE=lbry
       - MYSQL_USER=lbry
       - MYSQL_PASSWORD=lbry
       - MYSQL_LOG_CONSOLE=true

scripts/stress_test.py

@@ -0,0 +1,84 @@
+import sqlite3
+import time
+
+import faker
+from faker.providers import misc
+
+fake = faker.Faker()
+fake.add_provider(misc)
+
+
+if __name__ == '__main__':
+    song_time = """One, two, three!
+My baby don't mess around
+'Cause she loves me so
+This I know fo sho!
+But does she really wanna
+But can't stand to see me walk out tha door
+Don't try to fight the feeling
+Because the thought alone is killin' me right now
+Thank God for Mom and Dad
+For sticking to together
+Like we don't know how
+Hey ya! Hey ya!
+Hey ya! Hey ya!
+Hey ya! Hey ya!
+Hey ya! Hey ya!
+You think you've got it
+Oh, you think you've got it
+But got it just don't get it when there's nothin' at all
+We get together
+Oh, we get together
+But separate's always better when there's feelings involved
+Know what they say -its
+Nothing lasts forever!
+Then what makes it, then what makes it
+Then what makes it, then what makes it
+Then what makes love the exception?
+So why, oh, why, oh
+Why, oh, why, oh, why, oh
+Are we still in denial when we know we're not happy here
+Hey ya! (y'all don't want to here me, ya just want to dance) Hey ya!
+Don't want to meet your daddy (oh ohh), just want you in my caddy (oh ohh)
+Hey ya! (oh, oh!) Hey ya! (oh, oh!)
+Don't want to meet your momma, just want to make you cum-a (oh, oh!)
+I'm (oh, oh) I'm (oh, oh) I'm just being honest! (oh, oh)
+I'm just being honest!
+Hey! alright now! alright now, fellas!
+Yea?
+Now, what cooler than being cool?
+Ice cold!
+I can't hear ya! I say what's, what's cooler than being cool?
+Ice cold!
+Alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright!
+Okay, now ladies!
+Yea?
+Now we gonna break this thang down for just a few seconds
+Now don't have me break this thang down for nothin'
+I want to see you on your badest behavior!
+Lend me some sugar, I am your neighbor!
+Ah! Here we go now,
+Shake it, shake it, shake it, shake it, shake it
+Shake it, shake it, shake it, shake it
+Shake it like a Polaroid picture! Hey ya!
+Shake it, shake it, shake it, shake it, shake it
+Shake it, shake it, shake it, suga!
+Shake it like a Polaroid picture!
+Now all the Beyonce's, and Lucy Lu's, and baby dolls
+Get on tha floor get on tha floor!
+Shake it like a Polaroid picture!
+Oh, you! oh, you!
+Hey ya!(oh, oh) Hey ya!(oh, oh)
+Hey ya!(oh, oh) Hey ya!(oh, oh)
+Hey ya!(oh, oh) Hey ya!(oh, oh)"""
+
+    song = song_time.split('\n')
+    claim_id = '2aa106927b733e2602ffb565efaccc78c2ed89df'
+    run_len = [(fake.sha256(), song_time, claim_id, str(int(time.time()))) for k in range(5000)]
+
+    conn = sqlite3.connect('database/default_test.db')
+    with conn:
+        curs = conn.executemany("""
+        INSERT INTO COMMENT(CommentId, Body, LbryClaimId, Timestamp) VALUES (?, ?, ?, ?)
+        """, run_len)
+        print(f'rows changed: {curs.rowcount}')

scripts/valid_signatures.py

@@ -2,12 +2,11 @@ import binascii
 import logging
 import hashlib
 import json
-# todo: remove sqlite3 as a dependency
 import sqlite3
 import asyncio
 import aiohttp
 
-from src.server.validation import is_signature_valid, get_encoded_signature
+from server.validation import is_signature_valid, get_encoded_signature
 
 logger = logging.getLogger(__name__)
 

src/database/comments_ddl.sql

@@ -1,50 +1,76 @@
-USE `social`;
+PRAGMA FOREIGN_KEYS = ON;
-ALTER DATABASE `social`
-    DEFAULT CHARACTER SET utf8mb4
-    DEFAULT COLLATE utf8mb4_unicode_ci;
 
-DROP TABLE IF EXISTS `CHANNEL`;
+-- Although I know this file is unnecessary, I like keeping it around.
-CREATE TABLE `CHANNEL` (
-        `claimid` VARCHAR(40)  NOT NULL,
-        `name`  CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
-        CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`)
-    )
-CHARACTER SET utf8mb4
-COLLATE utf8mb4_unicode_ci;
 
-DROP TABLE IF EXISTS `COMMENT`;
+-- I'm not gonna remove it.
-CREATE TABLE `COMMENT` (
-        -- should be changed to CHAR(64)
-        `commentid`   CHAR(64)   NOT NULL,
-        -- should be changed to CHAR(40)
-        `lbryclaimid` CHAR(40)   NOT NULL,
-        -- can be null, so idk if this should be char(40)
-        `channelid`   CHAR(40)              DEFAULT NULL,
-        `body`        TEXT
-            CHARACTER SET utf8mb4
-            COLLATE utf8mb4_unicode_ci
-            NOT NULL,
-        `parentid`    CHAR(64)               DEFAULT NULL,
-        `signature`   CHAR(128)              DEFAULT NULL,
-        -- 22 chars long is prolly enough
-        `signingts`   VARCHAR(22)               DEFAULT NULL,
 
-        `timestamp`   INTEGER NOT NULL,
+-- tables
-        -- there's no way that the timestamp will ever reach 22 characters
+CREATE TABLE IF NOT EXISTS COMMENT
-        `ishidden`    BOOLEAN                    DEFAULT FALSE,
+(
-        CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`)
+    CommentId   TEXT    NOT NULL,
-         -- setting null implies comment is top level
+    LbryClaimId TEXT    NOT NULL,
-    )
+    ChannelId   TEXT             DEFAULT NULL,
-CHARACTER SET utf8mb4
+    Body        TEXT    NOT NULL,
-COLLATE utf8mb4_unicode_ci;
+    ParentId    TEXT             DEFAULT NULL,
+    Signature   TEXT             DEFAULT NULL,
+    Timestamp   INTEGER NOT NULL,
+    SigningTs   TEXT             DEFAULT NULL,
+    IsHidden    BOOLEAN NOT NULL DEFAULT FALSE,
+    CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
+    CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
+    CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
+        ON DELETE NO ACTION ON UPDATE NO ACTION,
+    CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId)
+        ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
+);
+
+-- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE);
+-- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL;
+
+-- DROP TABLE IF EXISTS CHANNEL;
+CREATE TABLE IF NOT EXISTS CHANNEL
+(
+    ClaimId TEXT NOT NULL,
+    Name    TEXT NOT NULL,
+    CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId)
+        ON CONFLICT IGNORE
+);
 
 
-ALTER TABLE COMMENT
+-- indexes
-    ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`)
+-- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX;
-        ON DELETE CASCADE ON UPDATE CASCADE,
+-- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
-    ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`)
-            ON UPDATE CASCADE ON DELETE CASCADE
-;
 
-CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`);
+-- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
-CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`);
+
+-- VIEWS
+CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS
+SELECT C.CommentId                                     AS comment_id,
+       C.Body                                          AS comment,
+       C.LbryClaimId                                   AS claim_id,
+       C.Timestamp                                     AS timestamp,
+       CHAN.Name                                       AS channel_name,
+       CHAN.ClaimId                                    AS channel_id,
+       ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
+       C.Signature                                     AS signature,
+       C.SigningTs                                     AS signing_ts,
+       C.ParentId                                      AS parent_id,
+       C.IsHidden                                      AS is_hidden
+FROM COMMENT AS C
+         LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
+ORDER BY C.Timestamp DESC;
+
+
+DROP VIEW IF EXISTS COMMENT_REPLIES;
+CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS
+SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body
+FROM COMMENT AS OG
+         JOIN COMMENT AS PARENT
+              ON OG.ParentId = PARENT.CommentId
+         JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId
+         JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId
+ORDER BY OG.Timestamp;
+
+-- this is the default channel for anyone who wants to publish anonymously
+-- INSERT INTO CHANNEL
+-- VALUES ('9cb713f01bf247a0e03170b5ed00d5161340c486', '@Anonymous');

src/database/models.py

@@ -13,25 +13,25 @@ from src.misc import clean
 
 
 class Channel(Model):
-    claim_id = FixedCharField(column_name='claimid', primary_key=True, max_length=40)
+    claim_id = CharField(column_name='ClaimId', primary_key=True, max_length=40)
-    name = CharField(column_name='name', max_length=255)
+    name = CharField(column_name='Name', max_length=256)
 
     class Meta:
         table_name = 'CHANNEL'
 
 
 class Comment(Model):
-    comment = TextField(column_name='body')
+    comment = CharField(column_name='Body', max_length=2000)
     channel = ForeignKeyField(
         backref='comments',
-        column_name='channelid',
+        column_name='ChannelId',
         field='claim_id',
         model=Channel,
         null=True
     )
-    comment_id = FixedCharField(column_name='commentid', primary_key=True, max_length=64)
+    comment_id = CharField(column_name='CommentId', primary_key=True, max_length=64)
-    is_hidden = BooleanField(column_name='ishidden', constraints=[SQL("DEFAULT 0")])
+    is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")])
-    claim_id = FixedCharField(max_length=40, column_name='lbryclaimid')
+    claim_id = CharField(max_length=40, column_name='LbryClaimId')
     parent = ForeignKeyField(
         column_name='ParentId',
         field='comment_id',
@@ -39,9 +39,9 @@ class Comment(Model):
         null=True,
         backref='replies'
     )
-    signature = FixedCharField(max_length=128, column_name='signature', null=True, unique=True)
+    signature = CharField(max_length=128, column_name='Signature', null=True, unique=True)
-    signing_ts = TextField(column_name='signingts', null=True)
+    signing_ts = TextField(column_name='SigningTs', null=True)
-    timestamp = IntegerField(column_name='timestamp')
+    timestamp = IntegerField(column_name='Timestamp')
 
     class Meta:
         table_name = 'COMMENT'

src/database/schema.py

@@ -0,0 +1,76 @@
+PRAGMAS = """
+    PRAGMA FOREIGN_KEYS = ON;
+"""
+
+CREATE_COMMENT_TABLE = """
+    CREATE TABLE IF NOT EXISTS COMMENT (
+        CommentId   TEXT    NOT NULL,
+        LbryClaimId TEXT    NOT NULL,
+        ChannelId   TEXT                DEFAULT NULL,
+        Body        TEXT    NOT NULL,
+        ParentId    TEXT                DEFAULT NULL,
+        Signature   TEXT                DEFAULT NULL,
+        Timestamp   INTEGER NOT NULL,
+        SigningTs   TEXT                DEFAULT NULL,
+        IsHidden    BOOLEAN NOT NULL    DEFAULT 0,
+        CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
+        CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
+        CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
+            ON DELETE NO ACTION ON UPDATE NO ACTION,
+        CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId)
+            ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
+    );
+"""
+
+CREATE_COMMENT_INDEXES = """
+    CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
+    CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
+"""
+
+CREATE_CHANNEL_TABLE = """
+    CREATE TABLE IF NOT EXISTS CHANNEL (
+        ClaimId TEXT NOT NULL,
+        Name    TEXT NOT NULL,
+        CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId)
+            ON CONFLICT IGNORE
+    );
+"""
+
+CREATE_COMMENTS_ON_CLAIMS_VIEW = """
+    CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT 
+        C.CommentId AS comment_id,
+        C.Body AS comment,
+        C.LbryClaimId AS claim_id,
+        C.Timestamp AS timestamp,
+        CHAN.Name AS channel_name,
+        CHAN.ClaimId AS channel_id,
+        ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
+        C.Signature AS signature,
+        C.SigningTs AS signing_ts,
+        C.ParentId AS parent_id,
+        C.IsHidden AS is_hidden
+    FROM COMMENT AS C
+             LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
+    ORDER BY C.Timestamp DESC;
+"""
+
+# not being used right now but should be kept around when Tom finally asks for replies
+CREATE_COMMENT_REPLIES_VIEW = """
+CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS
+SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body
+FROM COMMENT AS OG
+         JOIN COMMENT AS PARENT
+              ON OG.ParentId = PARENT.CommentId
+         JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId
+         JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId
+ORDER BY OG.Timestamp;
+"""
+
+CREATE_TABLES_QUERY = (
+        PRAGMAS +
+        CREATE_COMMENT_TABLE +
+        CREATE_COMMENT_INDEXES +
+        CREATE_CHANNEL_TABLE +
+        CREATE_COMMENTS_ON_CLAIMS_VIEW +
+        CREATE_COMMENT_REPLIES_VIEW
+)

src/database/writes.py

@@ -1,4 +1,3 @@
-# TODO: scrap notification routines from these files & supply them in handles
 import logging
 import sqlite3
 from asyncio import coroutine

src/main.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 import yaml
 import logging
 import logging.config

src/misc.py

@@ -16,7 +16,7 @@ async def get_claim_from_id(app, claim_id, **kwargs):
 
 def clean_input_params(kwargs: dict):
     for k, v in kwargs.items():
-        if type(v) is str and k != 'comment':
+        if type(v) is str and k is not 'comment':
             kwargs[k] = v.strip()
             if k in ID_LIST:
                 kwargs[k] = v.lower()

src/server/app.py

@@ -28,7 +28,6 @@ def setup_database(app):
             host=config[mode]['host'],
             password=config[mode]['password'],
             port=config[mode]['port'],
-            charset=config[mode]['charset'],
         )
     elif config[mode]['database'] == 'sqlite':
         app['db'] = SqliteDatabase(

src/server/errors.py

@@ -1,5 +1,3 @@
-import json
-
 import logging
 import aiohttp
 
@@ -34,11 +32,8 @@ def make_error(error, exc=None) -> dict:
 async def report_error(app, exc, body: dict):
     try:
         if 'slack_webhook' in app['config']:
-            body_dump = json.dumps(body, indent=4)
-            exec_name = type(exc).__name__
-            exec_body = str(exc)
             message = {
-                "text": f"Got `{exec_name}`: `\n{exec_body}`\n```{body_dump}```"
+                "text": f"Got `{type(exc).__name__}`: `\n{str(exc)}`\n```{body}```"
             }
             async with aiohttp.ClientSession() as sesh:
                 async with sesh.post(app['config']['slack_webhook'], json=message) as resp:

src/server/external.py

@@ -37,10 +37,6 @@ def create_notification_batch(action: str, comments: List[dict]) -> List[dict]:
         }
         if comment.get('channel_id'):
             event['channel_id'] = comment['channel_id']
-        if comment.get('parent_id'):
-            event['parent_id'] = comment['parent_id']
-        if comment.get('comment'):
-            event['comment'] = comment['comment']
         events.append(event)
     return events
 

src/server/handles.py

@@ -7,7 +7,6 @@ from aiohttp import web
 from aiojobs.aiohttp import atomic
 from peewee import DoesNotExist
 
-from src.server.external import send_notification
 from src.server.validation import validate_signature_from_claim
 from src.misc import clean_input_params, get_claim_from_id
 from src.server.errors import make_error, report_error
@@ -117,7 +116,7 @@ async def handle_abandon_comment(
     else:
         if not validate_signature_from_claim(channel, signature, signing_ts, comment_id):
             raise ValueError('Abandon signature could not be validated')
-    await app['webhooks'].spawn(send_notification(app, 'DELETE', comment))
+
     with app['db'].atomic():
         return {
             'abandoned': delete_comment(comment_id)
@@ -185,17 +184,15 @@ async def handle_edit_comment(app, comment: str = None, comment_id: str = None,
     with app['db'].atomic():
         if not edit_comment(comment_id, comment, signature, signing_ts):
             raise ValueError('Comment could not be edited')
-        updated_comment = get_comment(comment_id)
+        return get_comment(comment_id)
-        await app['webhooks'].spawn(send_notification(app, 'UPDATE', updated_comment))
-        return updated_comment
 
 
 # TODO: retrieve stake amounts for each channel & store in db
-async def handle_create_comment(app, comment: str = None, claim_id: str = None,
+def handle_create_comment(app, comment: str = None, claim_id: str = None,
                           parent_id: str = None, channel_id: str = None, channel_name: str = None,
                           signature: str = None, signing_ts: str = None) -> dict:
     with app['db'].atomic():
-        comment = create_comment(
+        return create_comment(
             comment=comment,
             claim_id=claim_id,
             parent_id=parent_id,
@@ -204,8 +201,6 @@ async def handle_create_comment(app, comment: str = None, claim_id: str = None,
             signature=signature,
             signing_ts=signing_ts
         )
-        await app['webhooks'].spawn(send_notification(app, 'CREATE', comment))
-        return comment
 
 
 METHODS = {