diff --git a/.travis.yml b/.travis.yml index fe9a4de..fe81b23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,5 +30,6 @@ jobs: name: "Unit Tests" install: - pip install -e . + - mkdir database script: - python -m unittest diff --git a/README.md b/README.md index cdeef1d..471f552 100644 --- a/README.md +++ b/README.md @@ -3,36 +3,34 @@ [](https://travis-ci.com/lbryio/comment-server) [](https://codeclimate.com/github/lbryio/comment-server/maintainability) +This is the code for the LBRY Comment Server. +Fork it, run it, set it on fire. Up to you. + ## Before Installing -Install the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) +Comment Deletion requires having the [`lbry-sdk`](https://github.com/lbryio/lbry-sdk) in order to validate & properly delete comments. - + + ## Installation #### Installing the server: ```bash -$ git clone https://github.com/lbryio/comment-server +$ git clone https://github.com/osilkin98/comment-server $ cd comment-server # create a virtual environment -$ virtualenv --python=python3.8 venv +$ virtualenv --python=python3 venv # Enter the virtual environment $ source venv/bin/activate -# Install required dependencies -(venv) $ pip install -e . - -# Run the server -(venv) $ python src/main.py \ - --port=5921 \ # use a different port besides the default - --config=conf.yml \ # provide a custom config file - & \ # detach and run the service in the background +# install the Server as a Executable Target +(venv) $ python setup.py develop ``` ### Installing the systemd Service Monitor @@ -72,11 +70,16 @@ To Test the database, simply run: There are basic tests to run against the server, though they require that there is a server instance running, though the database - chosen may have to be edited in `config/conf.yml`. + chosen may have to be edited in `config/conf.json`. Additionally there are HTTP requests that can be send with whatever software you choose to test the integrity of the comment server. +## Schema + + + + ## Contributing Contributions are welcome, verbosity is encouraged. Please be considerate diff --git a/config/conf.yml b/config/conf.yml index fc22322..6d90446 100644 --- a/config/conf.yml +++ b/config/conf.yml @@ -12,9 +12,8 @@ testing: # actual database should be running MySQL production: - charset: utf8mb4 database: mysql - name: social + name: lbry user: lbry password: lbry host: localhost diff --git a/docker-compose.yml b/docker-compose.yml index e5f9099..3faf5f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,12 +6,11 @@ services: mysql: image: mysql/mysql-server:5.7.27 restart: "no" - command: --character_set_server=utf8mb4 --max_allowed_packet=1073741824 ports: - "3306:3306" environment: - MYSQL_ALLOW_EMPTY_PASSWORD=true - - MYSQL_DATABASE=social + - MYSQL_DATABASE=lbry - MYSQL_USER=lbry - MYSQL_PASSWORD=lbry - MYSQL_LOG_CONSOLE=true diff --git a/scripts/stress_test.py b/scripts/stress_test.py new file mode 100644 index 0000000..6022396 --- /dev/null +++ b/scripts/stress_test.py @@ -0,0 +1,84 @@ +import sqlite3 +import time + +import faker +from faker.providers import misc + +fake = faker.Faker() +fake.add_provider(misc) + + +if __name__ == '__main__': + song_time = """One, two, three! +My baby don't mess around +'Cause she loves me so +This I know fo sho! +But does she really wanna +But can't stand to see me walk out tha door +Don't try to fight the feeling +Because the thought alone is killin' me right now +Thank God for Mom and Dad +For sticking to together +Like we don't know how +Hey ya! Hey ya! +Hey ya! Hey ya! +Hey ya! Hey ya! +Hey ya! Hey ya! +You think you've got it +Oh, you think you've got it +But got it just don't get it when there's nothin' at all +We get together +Oh, we get together +But separate's always better when there's feelings involved +Know what they say -its +Nothing lasts forever! +Then what makes it, then what makes it +Then what makes it, then what makes it +Then what makes love the exception? +So why, oh, why, oh +Why, oh, why, oh, why, oh +Are we still in denial when we know we're not happy here +Hey ya! (y'all don't want to here me, ya just want to dance) Hey ya! +Don't want to meet your daddy (oh ohh), just want you in my caddy (oh ohh) +Hey ya! (oh, oh!) Hey ya! (oh, oh!) +Don't want to meet your momma, just want to make you cum-a (oh, oh!) +I'm (oh, oh) I'm (oh, oh) I'm just being honest! (oh, oh) +I'm just being honest! +Hey! alright now! alright now, fellas! +Yea? +Now, what cooler than being cool? +Ice cold! +I can't hear ya! I say what's, what's cooler than being cool? +Ice cold! +Alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright alright! +Okay, now ladies! +Yea? +Now we gonna break this thang down for just a few seconds +Now don't have me break this thang down for nothin' +I want to see you on your badest behavior! +Lend me some sugar, I am your neighbor! +Ah! Here we go now, +Shake it, shake it, shake it, shake it, shake it +Shake it, shake it, shake it, shake it +Shake it like a Polaroid picture! Hey ya! +Shake it, shake it, shake it, shake it, shake it +Shake it, shake it, shake it, suga! +Shake it like a Polaroid picture! +Now all the Beyonce's, and Lucy Lu's, and baby dolls +Get on tha floor get on tha floor! +Shake it like a Polaroid picture! +Oh, you! oh, you! +Hey ya!(oh, oh) Hey ya!(oh, oh) +Hey ya!(oh, oh) Hey ya!(oh, oh) +Hey ya!(oh, oh) Hey ya!(oh, oh)""" + + song = song_time.split('\n') + claim_id = '2aa106927b733e2602ffb565efaccc78c2ed89df' + run_len = [(fake.sha256(), song_time, claim_id, str(int(time.time()))) for k in range(5000)] + + conn = sqlite3.connect('database/default_test.db') + with conn: + curs = conn.executemany(""" + INSERT INTO COMMENT(CommentId, Body, LbryClaimId, Timestamp) VALUES (?, ?, ?, ?) + """, run_len) + print(f'rows changed: {curs.rowcount}') diff --git a/scripts/valid_signatures.py b/scripts/valid_signatures.py index 4b7c641..19fc878 100644 --- a/scripts/valid_signatures.py +++ b/scripts/valid_signatures.py @@ -2,12 +2,11 @@ import binascii import logging import hashlib import json -# todo: remove sqlite3 as a dependency import sqlite3 import asyncio import aiohttp -from src.server.validation import is_signature_valid, get_encoded_signature +from server.validation import is_signature_valid, get_encoded_signature logger = logging.getLogger(__name__) diff --git a/src/database/comments_ddl.sql b/src/database/comments_ddl.sql index f488406..2dfc19d 100644 --- a/src/database/comments_ddl.sql +++ b/src/database/comments_ddl.sql @@ -1,50 +1,76 @@ -USE `social`; -ALTER DATABASE `social` - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; +PRAGMA FOREIGN_KEYS = ON; -DROP TABLE IF EXISTS `CHANNEL`; -CREATE TABLE `CHANNEL` ( - `claimid` VARCHAR(40) NOT NULL, - `name` CHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, - CONSTRAINT `channel_pk` PRIMARY KEY (`claimid`) - ) -CHARACTER SET utf8mb4 -COLLATE utf8mb4_unicode_ci; +-- Although I know this file is unnecessary, I like keeping it around. -DROP TABLE IF EXISTS `COMMENT`; -CREATE TABLE `COMMENT` ( - -- should be changed to CHAR(64) - `commentid` CHAR(64) NOT NULL, - -- should be changed to CHAR(40) - `lbryclaimid` CHAR(40) NOT NULL, - -- can be null, so idk if this should be char(40) - `channelid` CHAR(40) DEFAULT NULL, - `body` TEXT - CHARACTER SET utf8mb4 - COLLATE utf8mb4_unicode_ci - NOT NULL, - `parentid` CHAR(64) DEFAULT NULL, - `signature` CHAR(128) DEFAULT NULL, - -- 22 chars long is prolly enough - `signingts` VARCHAR(22) DEFAULT NULL, +-- I'm not gonna remove it. - `timestamp` INTEGER NOT NULL, - -- there's no way that the timestamp will ever reach 22 characters - `ishidden` BOOLEAN DEFAULT FALSE, - CONSTRAINT `COMMENT_PRIMARY_KEY` PRIMARY KEY (`commentid`) - -- setting null implies comment is top level - ) -CHARACTER SET utf8mb4 -COLLATE utf8mb4_unicode_ci; +-- tables +CREATE TABLE IF NOT EXISTS COMMENT +( + CommentId TEXT NOT NULL, + LbryClaimId TEXT NOT NULL, + ChannelId TEXT DEFAULT NULL, + Body TEXT NOT NULL, + ParentId TEXT DEFAULT NULL, + Signature TEXT DEFAULT NULL, + Timestamp INTEGER NOT NULL, + SigningTs TEXT DEFAULT NULL, + IsHidden BOOLEAN NOT NULL DEFAULT FALSE, + CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, + CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, + CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) + ON DELETE NO ACTION ON UPDATE NO ACTION, + CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId) + ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level +); + +-- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE); +-- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL; + +-- DROP TABLE IF EXISTS CHANNEL; +CREATE TABLE IF NOT EXISTS CHANNEL +( + ClaimId TEXT NOT NULL, + Name TEXT NOT NULL, + CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId) + ON CONFLICT IGNORE +); -ALTER TABLE COMMENT - ADD CONSTRAINT `comment_channel_fk` FOREIGN KEY (`channelid`) REFERENCES `CHANNEL` (`claimid`) - ON DELETE CASCADE ON UPDATE CASCADE, - ADD CONSTRAINT `comment_parent_fk` FOREIGN KEY (`parentid`) REFERENCES `COMMENT` (`commentid`) - ON UPDATE CASCADE ON DELETE CASCADE -; +-- indexes +-- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX; +-- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); -CREATE INDEX `claim_comment_index` ON `COMMENT` (`lbryclaimid`, `commentid`); -CREATE INDEX `channel_comment_index` ON `COMMENT` (`channelid`, `commentid`); +-- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); + +-- VIEWS +CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS +SELECT C.CommentId AS comment_id, + C.Body AS comment, + C.LbryClaimId AS claim_id, + C.Timestamp AS timestamp, + CHAN.Name AS channel_name, + CHAN.ClaimId AS channel_id, + ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, + C.Signature AS signature, + C.SigningTs AS signing_ts, + C.ParentId AS parent_id, + C.IsHidden AS is_hidden +FROM COMMENT AS C + LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId +ORDER BY C.Timestamp DESC; + + +DROP VIEW IF EXISTS COMMENT_REPLIES; +CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS +SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body +FROM COMMENT AS OG + JOIN COMMENT AS PARENT + ON OG.ParentId = PARENT.CommentId + JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId + JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId +ORDER BY OG.Timestamp; + +-- this is the default channel for anyone who wants to publish anonymously +-- INSERT INTO CHANNEL +-- VALUES ('9cb713f01bf247a0e03170b5ed00d5161340c486', '@Anonymous'); diff --git a/src/database/models.py b/src/database/models.py index b35f122..9c40586 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -13,25 +13,25 @@ from src.misc import clean class Channel(Model): - claim_id = FixedCharField(column_name='claimid', primary_key=True, max_length=40) - name = CharField(column_name='name', max_length=255) + claim_id = CharField(column_name='ClaimId', primary_key=True, max_length=40) + name = CharField(column_name='Name', max_length=256) class Meta: table_name = 'CHANNEL' class Comment(Model): - comment = TextField(column_name='body') + comment = CharField(column_name='Body', max_length=2000) channel = ForeignKeyField( backref='comments', - column_name='channelid', + column_name='ChannelId', field='claim_id', model=Channel, null=True ) - comment_id = FixedCharField(column_name='commentid', primary_key=True, max_length=64) - is_hidden = BooleanField(column_name='ishidden', constraints=[SQL("DEFAULT 0")]) - claim_id = FixedCharField(max_length=40, column_name='lbryclaimid') + comment_id = CharField(column_name='CommentId', primary_key=True, max_length=64) + is_hidden = BooleanField(column_name='IsHidden', constraints=[SQL("DEFAULT 0")]) + claim_id = CharField(max_length=40, column_name='LbryClaimId') parent = ForeignKeyField( column_name='ParentId', field='comment_id', @@ -39,9 +39,9 @@ class Comment(Model): null=True, backref='replies' ) - signature = FixedCharField(max_length=128, column_name='signature', null=True, unique=True) - signing_ts = TextField(column_name='signingts', null=True) - timestamp = IntegerField(column_name='timestamp') + signature = CharField(max_length=128, column_name='Signature', null=True, unique=True) + signing_ts = TextField(column_name='SigningTs', null=True) + timestamp = IntegerField(column_name='Timestamp') class Meta: table_name = 'COMMENT' diff --git a/src/database/schema.py b/src/database/schema.py new file mode 100644 index 0000000..c75681b --- /dev/null +++ b/src/database/schema.py @@ -0,0 +1,76 @@ +PRAGMAS = """ + PRAGMA FOREIGN_KEYS = ON; +""" + +CREATE_COMMENT_TABLE = """ + CREATE TABLE IF NOT EXISTS COMMENT ( + CommentId TEXT NOT NULL, + LbryClaimId TEXT NOT NULL, + ChannelId TEXT DEFAULT NULL, + Body TEXT NOT NULL, + ParentId TEXT DEFAULT NULL, + Signature TEXT DEFAULT NULL, + Timestamp INTEGER NOT NULL, + SigningTs TEXT DEFAULT NULL, + IsHidden BOOLEAN NOT NULL DEFAULT 0, + CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, + CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, + CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) + ON DELETE NO ACTION ON UPDATE NO ACTION, + CONSTRAINT COMMENT_PARENT_FK FOREIGN KEY (ParentId) REFERENCES COMMENT (CommentId) + ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level + ); +""" + +CREATE_COMMENT_INDEXES = """ + CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); + CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); +""" + +CREATE_CHANNEL_TABLE = """ + CREATE TABLE IF NOT EXISTS CHANNEL ( + ClaimId TEXT NOT NULL, + Name TEXT NOT NULL, + CONSTRAINT CHANNEL_PK PRIMARY KEY (ClaimId) + ON CONFLICT IGNORE + ); +""" + +CREATE_COMMENTS_ON_CLAIMS_VIEW = """ + CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT + C.CommentId AS comment_id, + C.Body AS comment, + C.LbryClaimId AS claim_id, + C.Timestamp AS timestamp, + CHAN.Name AS channel_name, + CHAN.ClaimId AS channel_id, + ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, + C.Signature AS signature, + C.SigningTs AS signing_ts, + C.ParentId AS parent_id, + C.IsHidden AS is_hidden + FROM COMMENT AS C + LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId + ORDER BY C.Timestamp DESC; +""" + +# not being used right now but should be kept around when Tom finally asks for replies +CREATE_COMMENT_REPLIES_VIEW = """ +CREATE VIEW IF NOT EXISTS COMMENT_REPLIES (Author, CommentBody, ParentAuthor, ParentCommentBody) AS +SELECT AUTHOR.Name, OG.Body, PCHAN.Name, PARENT.Body +FROM COMMENT AS OG + JOIN COMMENT AS PARENT + ON OG.ParentId = PARENT.CommentId + JOIN CHANNEL AS PCHAN ON PARENT.ChannelId = PCHAN.ClaimId + JOIN CHANNEL AS AUTHOR ON OG.ChannelId = AUTHOR.ClaimId +ORDER BY OG.Timestamp; +""" + +CREATE_TABLES_QUERY = ( + PRAGMAS + + CREATE_COMMENT_TABLE + + CREATE_COMMENT_INDEXES + + CREATE_CHANNEL_TABLE + + CREATE_COMMENTS_ON_CLAIMS_VIEW + + CREATE_COMMENT_REPLIES_VIEW +) diff --git a/src/database/writes.py b/src/database/writes.py index b86e31b..61caa78 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -1,4 +1,3 @@ -# TODO: scrap notification routines from these files & supply them in handles import logging import sqlite3 from asyncio import coroutine diff --git a/src/main.py b/src/main.py index d4b5a53..817dac4 100644 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,5 @@ import argparse +import json import yaml import logging import logging.config diff --git a/src/misc.py b/src/misc.py index 7c3d4bf..e9d5be8 100644 --- a/src/misc.py +++ b/src/misc.py @@ -16,7 +16,7 @@ async def get_claim_from_id(app, claim_id, **kwargs): def clean_input_params(kwargs: dict): for k, v in kwargs.items(): - if type(v) is str and k != 'comment': + if type(v) is str and k is not 'comment': kwargs[k] = v.strip() if k in ID_LIST: kwargs[k] = v.lower() diff --git a/src/server/app.py b/src/server/app.py index 37e89da..389083e 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -28,7 +28,6 @@ def setup_database(app): host=config[mode]['host'], password=config[mode]['password'], port=config[mode]['port'], - charset=config[mode]['charset'], ) elif config[mode]['database'] == 'sqlite': app['db'] = SqliteDatabase( diff --git a/src/server/errors.py b/src/server/errors.py index 644ce9e..273e4bd 100644 --- a/src/server/errors.py +++ b/src/server/errors.py @@ -1,5 +1,3 @@ -import json - import logging import aiohttp @@ -34,11 +32,8 @@ def make_error(error, exc=None) -> dict: async def report_error(app, exc, body: dict): try: if 'slack_webhook' in app['config']: - body_dump = json.dumps(body, indent=4) - exec_name = type(exc).__name__ - exec_body = str(exc) message = { - "text": f"Got `{exec_name}`: `\n{exec_body}`\n```{body_dump}```" + "text": f"Got `{type(exc).__name__}`: `\n{str(exc)}`\n```{body}```" } async with aiohttp.ClientSession() as sesh: async with sesh.post(app['config']['slack_webhook'], json=message) as resp: diff --git a/src/server/external.py b/src/server/external.py index 25bd634..aa0e7ee 100644 --- a/src/server/external.py +++ b/src/server/external.py @@ -37,10 +37,6 @@ def create_notification_batch(action: str, comments: List[dict]) -> List[dict]: } if comment.get('channel_id'): event['channel_id'] = comment['channel_id'] - if comment.get('parent_id'): - event['parent_id'] = comment['parent_id'] - if comment.get('comment'): - event['comment'] = comment['comment'] events.append(event) return events diff --git a/src/server/handles.py b/src/server/handles.py index a7fe43b..1c56663 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -7,7 +7,6 @@ from aiohttp import web from aiojobs.aiohttp import atomic from peewee import DoesNotExist -from src.server.external import send_notification from src.server.validation import validate_signature_from_claim from src.misc import clean_input_params, get_claim_from_id from src.server.errors import make_error, report_error @@ -117,7 +116,7 @@ async def handle_abandon_comment( else: if not validate_signature_from_claim(channel, signature, signing_ts, comment_id): raise ValueError('Abandon signature could not be validated') - await app['webhooks'].spawn(send_notification(app, 'DELETE', comment)) + with app['db'].atomic(): return { 'abandoned': delete_comment(comment_id) @@ -185,17 +184,15 @@ async def handle_edit_comment(app, comment: str = None, comment_id: str = None, with app['db'].atomic(): if not edit_comment(comment_id, comment, signature, signing_ts): raise ValueError('Comment could not be edited') - updated_comment = get_comment(comment_id) - await app['webhooks'].spawn(send_notification(app, 'UPDATE', updated_comment)) - return updated_comment + return get_comment(comment_id) # TODO: retrieve stake amounts for each channel & store in db -async def handle_create_comment(app, comment: str = None, claim_id: str = None, +def handle_create_comment(app, comment: str = None, claim_id: str = None, parent_id: str = None, channel_id: str = None, channel_name: str = None, signature: str = None, signing_ts: str = None) -> dict: with app['db'].atomic(): - comment = create_comment( + return create_comment( comment=comment, claim_id=claim_id, parent_id=parent_id, @@ -204,8 +201,6 @@ async def handle_create_comment(app, comment: str = None, claim_id: str = None, signature=signature, signing_ts=signing_ts ) - await app['webhooks'].spawn(send_notification(app, 'CREATE', comment)) - return comment METHODS = {