From 87150d94703ca35d552e9f519ea948f8784d639e Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sat, 3 Aug 2019 23:32:37 -0400 Subject: [PATCH 01/21] Adds `is_hidden` to schema --- src/database/comments_ddl.sql | 45 ++++++++++++++++++----------------- src/database/queries.py | 18 +++++++++----- src/database/schema.py | 12 ++++++---- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/database/comments_ddl.sql b/src/database/comments_ddl.sql index 7556fbd..fc85acf 100644 --- a/src/database/comments_ddl.sql +++ b/src/database/comments_ddl.sql @@ -9,12 +9,13 @@ CREATE TABLE IF NOT EXISTS COMMENT ( CommentId TEXT NOT NULL, LbryClaimId TEXT NOT NULL, - ChannelId TEXT DEFAULT NULL, + ChannelId TEXT DEFAULT (NULL), Body TEXT NOT NULL, - ParentId TEXT DEFAULT NULL, - Signature TEXT DEFAULT NULL, + ParentId TEXT DEFAULT (NULL), + Signature TEXT DEFAULT (NULL), Timestamp INTEGER NOT NULL, - SigningTs TEXT DEFAULT NULL, + SigningTs TEXT DEFAULT (NULL), + IsHidden BOOLEAN NOT NULL DEFAULT (FALSE), CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) @@ -23,6 +24,7 @@ CREATE TABLE IF NOT EXISTS COMMENT ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level ); +-- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE); -- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL; -- DROP TABLE IF EXISTS CHANNEL; @@ -37,27 +39,26 @@ CREATE TABLE IF NOT EXISTS CHANNEL -- indexes -- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX; -CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); +-- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); -CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); +-- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); -- VIEWS -CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS (comment_id, claim_id, timestamp, channel_name, channel_id, channel_url, - signature, signing_ts, parent_id, comment) AS -SELECT C.CommentId, - C.LbryClaimId, - C.Timestamp, - CHAN.Name, - CHAN.ClaimId, - 'lbry://' || CHAN.Name || '#' || CHAN.ClaimId, - C.Signature, - C.SigningTs, - C.ParentId, - C.Body -FROM COMMENT AS C - LEFT OUTER JOIN CHANNEL CHAN on C.ChannelId = CHAN.ClaimId -ORDER BY C.Timestamp DESC; - +CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT + C.CommentId AS comment_id, + C.Body AS comment, + C.LbryClaimId AS claim_id, + C.Timestamp AS timestamp, + CHAN.Name AS channel_name, + CHAN.ClaimId AS channel_id, + ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, + C.Signature AS signature, + C.SigningTs AS signing_ts, + C.ParentId AS parent_id, + C.IsHidden AS is_hidden + FROM COMMENT AS C + LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId + ORDER BY C.Timestamp DESC; DROP VIEW IF EXISTS COMMENT_REPLIES; diff --git a/src/database/queries.py b/src/database/queries.py index 417cf15..d26be9e 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -13,6 +13,8 @@ logger = logging.getLogger(__name__) def clean(thing: dict) -> dict: + if 'is_hidden' in thing: + thing.update({'is_hidden': bool(thing['is_hidden'])}) return {k: v for k, v in thing.items() if v} @@ -29,7 +31,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = if top_level: results = [clean(dict(row)) for row in conn.execute( """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id + channel_url, timestamp, signature, signing_ts, parent_id, is_hidden FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ? """, @@ -45,7 +47,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = elif parent_id is None: results = [clean(dict(row)) for row in conn.execute( """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id + channel_url, timestamp, signature, signing_ts, parent_id, is_hidden FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? LIMIT ? OFFSET ? """, @@ -61,7 +63,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = else: results = [clean(dict(row)) for row in conn.execute( """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id + channel_url, timestamp, signature, signing_ts, parent_id, is_hidden FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? """, @@ -105,7 +107,7 @@ def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict: with conn: curry = conn.execute( """ - SELECT comment, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id + SELECT comment, claim_id, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id, is_hidden FROM COMMENTS_ON_CLAIMS WHERE comment_id = ? """, (comment_id,) @@ -144,7 +146,11 @@ def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]: placeholders = ', '.join('?' for _ in comment_ids) with conn: return [clean(dict(row)) for row in conn.execute( - f'SELECT * FROM COMMENTS_ON_CLAIMS WHERE comment_id IN ({placeholders})', + """ + SELECT comment, claim_id, comment_id, channel_name, channel_id, + channel_url, timestamp, signature, signing_ts, parent_id, is_hidden + FROM COMMENTS_ON_CLAIMS + """ + f' WHERE comment_id IN ({placeholders})', tuple(comment_ids) )] @@ -178,7 +184,7 @@ def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str): SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ? """, (comment_id,) ).fetchone() - return dict(channel) if channel else dict() + return dict(channel) if channel else {} class DatabaseWriter(object): diff --git a/src/database/schema.py b/src/database/schema.py index bf46f72..f474765 100644 --- a/src/database/schema.py +++ b/src/database/schema.py @@ -6,12 +6,13 @@ CREATE_COMMENT_TABLE = """ CREATE TABLE IF NOT EXISTS COMMENT ( CommentId TEXT NOT NULL, LbryClaimId TEXT NOT NULL, - ChannelId TEXT DEFAULT NULL, + ChannelId TEXT DEFAULT NULL, Body TEXT NOT NULL, - ParentId TEXT DEFAULT NULL, - Signature TEXT DEFAULT NULL, + ParentId TEXT DEFAULT NULL, + Signature TEXT DEFAULT NULL, Timestamp INTEGER NOT NULL, - SigningTs TEXT DEFAULT NULL, + SigningTs TEXT DEFAULT NULL, + IsHidden BOOLEAN NOT NULL DEFAULT (FALSE), CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) @@ -46,7 +47,8 @@ CREATE_COMMENTS_ON_CLAIMS_VIEW = """ ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, C.Signature AS signature, C.SigningTs AS signing_ts, - C.ParentId AS parent_id + C.ParentId AS parent_id, + C.IsHidden as is_hidden FROM COMMENT AS C LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId ORDER BY C.Timestamp DESC; From 3c9e9d13c91ed323c9c22be3c4cd6a392180fcc4 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sat, 3 Aug 2019 23:34:00 -0400 Subject: [PATCH 02/21] Adds hide comment query & function --- src/database/queries.py | 9 +++++++++ src/database/writes.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/database/queries.py b/src/database/queries.py index d26be9e..568bc35 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -170,6 +170,15 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str): return bool(curs.rowcount) +def hide_comment_by_id(conn: sqlite3.Connection, comment_id: str): + with conn: + curs = conn.execute(""" + UPDATE OR IGNORE COMMENT SET IsHidden = TRUE + WHERE CommentId = ? + """, (comment_id,)) + return bool(curs.rowcount) + + def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str): with conn: conn.execute( diff --git a/src/database/writes.py b/src/database/writes.py index 0c4ba03..7fe99c2 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -4,11 +4,13 @@ import sqlite3 from asyncio import coroutine from database.queries import delete_comment_by_id -from src.server.misc import is_authentic_delete_signal +from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim from database.queries import get_comment_or_none from database.queries import insert_comment from database.queries import insert_channel +from database.queries import get_channel_id_from_comment_id +from database.queries import hide_comment_by_id from src.server.misc import channel_matches_pattern_or_error logger = logging.getLogger(__name__) @@ -47,5 +49,30 @@ async def delete_comment_if_authorized(app, comment_id, **kwargs): return {'deleted': await job.wait()} -async def write_comment(app, comment): - return await coroutine(create_comment_or_error)(app['writer'], **comment) +async def write_comment(app, params): + return await coroutine(create_comment_or_error)(app['writer'], **params) + + +async def hide_comment(app, comment_id): + return await coroutine(hide_comment_by_id)(app['writer'], comment_id) + + +# comment_ids: [ +# { +# "comment_id": id, +# "signing_ts": signing_ts, +# "signature": signature +# }, +# ... +# ] +async def hide_comment_if_authorized(app, comment_id, signing_ts, signature): + channel = get_channel_id_from_comment_id(app['reader'], comment_id) + claim = await request_lbrynet(app, 'claim_search', claim_id=channel['channel_id']) + claim = claim['items'][0] + if not validate_signature_from_claim(claim, signature, signing_ts, comment_id): + raise ValueError('Invalid Signature') + + job = await app['comment_scheduler'].spawn(hide_comment(app, comment_id)) + return { + 'hidden': await job.wait() + } From 26c01930ef23228d70162302b41876636711ff2a Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sat, 3 Aug 2019 23:35:13 -0400 Subject: [PATCH 03/21] Adds handles for hiding comments & verifying message validity --- src/server/handles.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/server/handles.py b/src/server/handles.py index f1d8f98..0ce8349 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -13,7 +13,10 @@ from database.queries import get_channel_id_from_comment_id from src.server.misc import is_valid_base_comment from src.server.misc import is_valid_credential_input from src.server.misc import make_error -from database.writes import delete_comment_if_authorized, write_comment +from database.writes import delete_comment_if_authorized +from database.writes import write_comment +from database.writes import hide_comment_if_authorized + logger = logging.getLogger(__name__) @@ -55,6 +58,10 @@ async def handle_delete_comment(app, params): return await delete_comment_if_authorized(app, **params) +async def handle_hide_comment(app, params): + return await hide_comment_if_authorized(app, **params) + + METHODS = { 'ping': ping, 'get_claim_comments': handle_get_claim_comments, @@ -63,6 +70,7 @@ METHODS = { 'get_channel_from_comment_id': handle_get_channel_from_comment_id, 'create_comment': handle_create_comment, 'delete_comment': handle_delete_comment, + 'hide_comment': handle_hide_comment, # 'abandon_comment': handle_delete_comment, } From 8d38bbd7ec0a6a6a805060267c897cedd6498162 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sat, 3 Aug 2019 23:35:37 -0400 Subject: [PATCH 04/21] Adds templates for querying lbrynet & validating signatures --- src/server/misc.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/server/misc.py b/src/server/misc.py index d0efd66..07567b9 100644 --- a/src/server/misc.py +++ b/src/server/misc.py @@ -41,20 +41,19 @@ def make_error(error, exc=None) -> dict: return body -async def resolve_channel_claim(app, channel_id, channel_name): - lbry_url = f'lbry://{channel_name}#{channel_id}' - resolve_body = {'method': 'resolve', 'params': {'urls': [lbry_url]}} +async def request_lbrynet(app, method, **params): + body = {'method': method, 'params': {**params}} try: - async with aiohttp.request('POST', app['config']['LBRYNET'], json=resolve_body) as req: + async with aiohttp.request('POST', app['config']['LBRYNET'], json=body) as req: try: resp = await req.json() except JSONDecodeError as jde: logger.exception(jde.msg) - raise Exception('JSON Decode Error in Claim Resolution') + raise Exception('JSON Decode Error In lbrynet request') finally: if 'result' in resp: - return resp['result'].get(lbry_url) - raise ValueError('claim resolution yields error', {'error': resp['error']}) + return resp['result'] + raise ValueError('LBRYNET Request Error', {'error': resp['error']}) except (ConnectionRefusedError, ClientConnectorError): logger.critical("Connection to the LBRYnet daemon failed, make sure it's running.") raise Exception("Server cannot verify delete signature") @@ -111,7 +110,8 @@ def is_valid_credential_input(channel_id=None, channel_name=None, signature=None async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id, signature, signing_ts): - claim = await resolve_channel_claim(app, channel_id, channel_name) + lbry_url = f'lbry://{channel_name}#{channel_id}' + claim = await request_lbrynet(app, 'resolve', urls=[lbry_url]) if claim: public_key = claim['value']['public_key'] claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1] @@ -124,6 +124,21 @@ async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id, return False +def validate_signature_from_claim(claim, signature, signing_ts, data: str): + try: + if claim: + public_key = claim['value']['public_key'] + claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1] + injest = b''.join((signing_ts.encode(), claim_hash, data.encode())) + return is_signature_valid( + encoded_signature=get_encoded_signature(signature), + signature_digest=hashlib.sha256(injest).digest(), + public_key_bytes=binascii.unhexlify(public_key.encode()) + ) + except: + return False + + def clean_input_params(kwargs: dict): for k, v in kwargs.items(): if type(v) is str: From 7f98417d9d561285ba186b5920c5ed93a55ddbff Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 17:45:15 -0400 Subject: [PATCH 05/21] Adds --- src/database/queries.py | 85 +++++++++++------------------------------ 1 file changed, 23 insertions(+), 62 deletions(-) diff --git a/src/database/queries.py b/src/database/queries.py index 568bc35..2dca72b 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -12,6 +12,13 @@ from src.database.schema import CREATE_TABLES_QUERY logger = logging.getLogger(__name__) +SELECT_COMMENTS_ON_CLAIMS = """ + SELECT comment, comment_id, channel_name, channel_id, channel_url, + timestamp, signature, signing_ts, parent_id, is_hidden + FROM COMMENTS_ON_CLAIMS +""" + + def clean(thing: dict) -> dict: if 'is_hidden' in thing: thing.update({'is_hidden': bool(thing['is_hidden'])}) @@ -30,51 +37,30 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = with conn: if top_level: results = [clean(dict(row)) for row in conn.execute( - """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id, is_hidden - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id IS NULL - LIMIT ? OFFSET ? """, + SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?", (claim_id, page_size, page_size * (page - 1)) )] count = conn.execute( - """ - SELECT COUNT(*) - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id IS NULL - """, (claim_id,) + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id IS NULL", + (claim_id,) ) elif parent_id is None: results = [clean(dict(row)) for row in conn.execute( - """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id, is_hidden - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? - LIMIT ? OFFSET ? """, + SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ", (claim_id, page_size, page_size * (page - 1)) )] count = conn.execute( - """ - SELECT COUNT(*) - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? - """, (claim_id,) + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ?", + (claim_id,) ) else: results = [clean(dict(row)) for row in conn.execute( - """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id, is_hidden - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id = ? - LIMIT ? OFFSET ? """, + SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? ", (claim_id, parent_id, page_size, page_size * (page - 1)) )] count = conn.execute( - """ - SELECT COUNT(*) - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id = ? - """, (claim_id, parent_id) + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id = ?", + (claim_id, parent_id) ) count = tuple(count.fetchone())[0] return { @@ -105,13 +91,7 @@ def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict: with conn: - curry = conn.execute( - """ - SELECT comment, claim_id, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id, is_hidden - FROM COMMENTS_ON_CLAIMS WHERE comment_id = ? - """, - (comment_id,) - ) + curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,)) thing = curry.fetchone() return clean(dict(thing)) if thing else None @@ -146,24 +126,11 @@ def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]: placeholders = ', '.join('?' for _ in comment_ids) with conn: return [clean(dict(row)) for row in conn.execute( - """ - SELECT comment, claim_id, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id, is_hidden - FROM COMMENTS_ON_CLAIMS - """ + f' WHERE comment_id IN ({placeholders})', + SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})', tuple(comment_ids) )] -def delete_anonymous_comment_by_id(conn: sqlite3.Connection, comment_id: str): - with conn: - curs = conn.execute( - "DELETE FROM COMMENT WHERE ChannelId IS NULL AND CommentId = ?", - (comment_id,) - ) - return curs.rowcount - - def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str): with conn: curs = conn.execute("DELETE FROM COMMENT WHERE CommentId = ?", (comment_id,)) @@ -172,26 +139,20 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str): def hide_comment_by_id(conn: sqlite3.Connection, comment_id: str): with conn: - curs = conn.execute(""" - UPDATE OR IGNORE COMMENT SET IsHidden = TRUE - WHERE CommentId = ? - """, (comment_id,)) + curs = conn.execute("UPDATE OR IGNORE COMMENT SET IsHidden = TRUE WHERE CommentId = ?", (comment_id,)) return bool(curs.rowcount) def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str): with conn: - conn.execute( - 'INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', - (channel_id, channel_name) - ) + curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', (channel_id, channel_name)) + return bool(curs.rowcount) def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str): with conn: - channel = conn.execute(""" - SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ? - """, (comment_id,) + channel = conn.execute( + "SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?", (comment_id,) ).fetchone() return dict(channel) if channel else {} From 1b79aefb054411b526ac82367c83d9b4d773fe1c Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 17:46:33 -0400 Subject: [PATCH 06/21] Adds hidden comment queries --- src/database/queries.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/database/queries.py b/src/database/queries.py index 2dca72b..7dd780c 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -22,7 +22,7 @@ SELECT_COMMENTS_ON_CLAIMS = """ def clean(thing: dict) -> dict: if 'is_hidden' in thing: thing.update({'is_hidden': bool(thing['is_hidden'])}) - return {k: v for k, v in thing.items() if v} + return {k: v for k, v in thing.items() if v is not None} def obtain_connection(filepath: str = None, row_factory: bool = True): @@ -68,10 +68,42 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = 'page': page, 'page_size': page_size, 'total_pages': math.ceil(count / page_size), - 'total_items': count + 'total_items': count, + 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id) } +def get_hidden_claim_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50): + with conn: + results = conn.execute( + SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden = ? LIMIT ? OFFSET ?", + (claim_id, hidden, page_size, page_size * (page - 1)) + ) + count = conn.execute( + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = ?", (claim_id, hidden) + ) + results = [clean(dict(row)) for row in results.fetchall()] + count = tuple(count.fetchone())[0] + + return { + 'items': results, + 'page': page, + 'page_size': page_size, + 'total_pages': math.ceil(count/page_size), + 'total_items': count, + 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id) + } + + +def claim_has_hidden_comments(conn, claim_id): + with conn: + result = conn.execute( + "SELECT COUNT(DISTINCT is_hidden) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = TRUE", + (claim_id,) + ) + return bool(tuple(result.fetchone())[0]) + + def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent_id: str = None, channel_id: str = None, signature: str = None, signing_ts: str = None) -> str: timestamp = int(time.time()) From c56562e7eaca20e8ff95747a6024e684a2a8ae40 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 17:47:11 -0400 Subject: [PATCH 07/21] Import formats --- src/database/writes.py | 12 ++++++------ src/server/handles.py | 13 +++++++------ tests/database_test.py | 11 +++++++---- tests/testcase.py | 2 +- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/database/writes.py b/src/database/writes.py index 7fe99c2..df0d950 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -3,14 +3,14 @@ import sqlite3 from asyncio import coroutine -from database.queries import delete_comment_by_id +from src.database.queries import delete_comment_by_id from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim -from database.queries import get_comment_or_none -from database.queries import insert_comment -from database.queries import insert_channel -from database.queries import get_channel_id_from_comment_id -from database.queries import hide_comment_by_id +from src.database.queries import get_comment_or_none +from src.database.queries import insert_comment +from src.database.queries import insert_channel +from src.database.queries import get_channel_id_from_comment_id +from src.database.queries import hide_comment_by_id from src.server.misc import channel_matches_pattern_or_error logger = logging.getLogger(__name__) diff --git a/src/server/handles.py b/src/server/handles.py index 0ce8349..af765ba 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -7,15 +7,16 @@ from aiohttp import web from aiojobs.aiohttp import atomic from src.server.misc import clean_input_params -from database.queries import get_claim_comments -from database.queries import get_comments_by_id, get_comment_ids -from database.queries import get_channel_id_from_comment_id +from src.database.queries import get_claim_comments +from src.database.queries import get_comments_by_id, get_comment_ids +from src.database.queries import get_channel_id_from_comment_id +from src.database.queries import get_hidden_claim_comments from src.server.misc import is_valid_base_comment from src.server.misc import is_valid_credential_input from src.server.misc import make_error -from database.writes import delete_comment_if_authorized -from database.writes import write_comment -from database.writes import hide_comment_if_authorized +from src.database.writes import delete_comment_if_authorized +from src.database.writes import write_comment +from src.database.writes import hide_comment_if_authorized logger = logging.getLogger(__name__) diff --git a/tests/database_test.py b/tests/database_test.py index 3de9312..4d9068c 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -5,10 +5,13 @@ from faker.providers import internet from faker.providers import lorem from faker.providers import misc -from database.queries import get_comments_by_id -from database.queries import get_comment_ids -from database.queries import get_claim_comments -from database.writes import create_comment_or_error +from src.database.queries import get_comments_by_id +from src.database.queries import get_comment_ids +from src.database.queries import get_claim_comments +from src.database.queries import get_hidden_claim_comments +from src.database.writes import create_comment_or_error +from src.database.queries import hide_comment_by_id +from src.database.queries import delete_comment_by_id from tests.testcase import DatabaseTestCase fake = faker.Faker() diff --git a/tests/testcase.py b/tests/testcase.py index 2357347..ddd8b44 100644 --- a/tests/testcase.py +++ b/tests/testcase.py @@ -6,7 +6,7 @@ from unittest.case import _Outcome import asyncio -from database.queries import obtain_connection, setup_database +from src.database.queries import obtain_connection, setup_database class AsyncioTestCase(unittest.TestCase): From 2b7e1df091c52d8ad00be6428d0bc2550487c437 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 17:47:53 -0400 Subject: [PATCH 08/21] Adds get_hidden_claim_comments handle & activates abandon_comment func --- src/server/handles.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/server/handles.py b/src/server/handles.py index af765ba..662e13b 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -28,23 +28,23 @@ def ping(*args): def handle_get_channel_from_comment_id(app, kwargs: dict): - with app['reader'] as conn: - return get_channel_id_from_comment_id(conn, **kwargs) + return get_channel_id_from_comment_id(app['reader'], **kwargs) def handle_get_comment_ids(app, kwargs): - with app['reader'] as conn: - return get_comment_ids(conn, **kwargs) + return get_comment_ids(app['reader'], **kwargs) def handle_get_claim_comments(app, kwargs): - with app['reader'] as conn: - return get_claim_comments(conn, **kwargs) + return get_claim_comments(app['reader'], **kwargs) def handle_get_comments_by_id(app, kwargs): - with app['reader'] as conn: - return get_comments_by_id(conn, **kwargs) + return get_comments_by_id(app['reader'], **kwargs) + + +def handle_get_hidden_claim_comments(app, kwargs): + return get_hidden_claim_comments(app['reader'], **kwargs) async def handle_create_comment(app, params): @@ -66,13 +66,14 @@ async def handle_hide_comment(app, params): METHODS = { 'ping': ping, 'get_claim_comments': handle_get_claim_comments, + 'get_hidden_claim_comments': handle_get_hidden_claim_comments, 'get_comment_ids': handle_get_comment_ids, 'get_comments_by_id': handle_get_comments_by_id, 'get_channel_from_comment_id': handle_get_channel_from_comment_id, 'create_comment': handle_create_comment, 'delete_comment': handle_delete_comment, + 'abandon_comment': handle_delete_comment, 'hide_comment': handle_hide_comment, - # 'abandon_comment': handle_delete_comment, } From 67d503acdbaac5d0b9353b5ab243ab2d5992e357 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 17:48:32 -0400 Subject: [PATCH 09/21] Adds unit tests for hidden comments + deleted comments --- tests/database_test.py | 68 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/database_test.py b/tests/database_test.py index 4d9068c..02050a0 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -20,7 +20,7 @@ fake.add_provider(lorem) fake.add_provider(misc) -class TestCommentCreation(DatabaseTestCase): +class TestDatabaseOperations(DatabaseTestCase): def setUp(self) -> None: super().setUp() self.claimId = '529357c3422c6046d3fec76be2358004ba22e340' @@ -195,6 +195,31 @@ class TestCommentCreation(DatabaseTestCase): self.assertLessEqual(len(replies), 50) self.assertEqual(len(replies), len(comments_ids)) + def test07HideComments(self): + comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') + comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + self.assertFalse(comment['is_hidden']) + success = hide_comment_by_id(self.conn, comm['comment_id']) + self.assertTrue(success) + comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + self.assertTrue(comment['is_hidden']) + success = hide_comment_by_id(self.conn, comm['comment_id']) + self.assertTrue(success) + comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + self.assertTrue(comment['is_hidden']) + + def test08DeleteComments(self): + comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') + comments = get_claim_comments(self.conn, self.claimId) + self.assertIn(comm, comments['items']) + deleted = delete_comment_by_id(self.conn, comm['comment_id']) + self.assertTrue(deleted) + comments = get_claim_comments(self.conn, self.claimId) + self.assertNotIn(comm, comments['items']) + deleted = delete_comment_by_id(self.conn, comm['comment_id']) + self.assertFalse(deleted) + + class ListDatabaseTest(DatabaseTestCase): def setUp(self) -> None: @@ -207,6 +232,8 @@ class ListDatabaseTest(DatabaseTestCase): comments = get_claim_comments(self.conn, claim_id) self.assertIsNotNone(comments) self.assertGreater(comments['page_size'], 0) + self.assertIn('has_hidden_comments', comments) + self.assertFalse(comments['has_hidden_comments']) top_comments = get_claim_comments(self.conn, claim_id, top_level=True, page=1, page_size=50) self.assertIsNotNone(top_comments) self.assertEqual(top_comments['page_size'], 50) @@ -221,6 +248,45 @@ class ListDatabaseTest(DatabaseTestCase): self.assertIsNotNone(matching_comments) self.assertEqual(len(matching_comments), len(comment_ids)) + def testHiddenCommentLists(self): + claim_id = 'a'*40 + comm1 = create_comment_or_error(self.conn, 'Comment #1', claim_id, '1'*40, '@Doge123', 'a'*128, '123') + comm2 = create_comment_or_error(self.conn, 'Comment #2', claim_id, '1'*40, '@Doge123', 'b'*128, '123') + comm3 = create_comment_or_error(self.conn, 'Comment #3', claim_id, '1'*40, '@Doge123', 'c'*128, '123') + comments = [comm1, comm2, comm3] + + comment_list = get_claim_comments(self.conn, claim_id) + self.assertIn('items', comment_list) + self.assertIn('has_hidden_comments', comment_list) + self.assertEqual(len(comments), comment_list['total_items']) + self.assertIn('has_hidden_comments', comment_list) + self.assertFalse(comment_list['has_hidden_comments']) + hide_comment_by_id(self.conn, comm2['comment_id']) + + default_comments = get_hidden_claim_comments(self.conn, claim_id) + self.assertIn('has_hidden_comments', default_comments) + + hidden_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=True) + self.assertIn('has_hidden_comments', hidden_comments) + self.assertEqual(default_comments, hidden_comments) + + hidden_comment = hidden_comments['items'][0] + self.assertEqual(hidden_comment['comment_id'], comm2['comment_id']) + + visible_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=False) + self.assertIn('has_hidden_comments', visible_comments) + self.assertNotIn(hidden_comment, visible_comments['items']) + + hidden_ids = [c['comment_id'] for c in hidden_comments['items']] + visible_ids = [c['comment_id'] for c in visible_comments['items']] + composite_ids = hidden_ids + visible_ids + composite_ids.sort() + + comment_list = get_claim_comments(self.conn, claim_id) + all_ids = [c['comment_id'] for c in comment_list['items']] + all_ids.sort() + self.assertEqual(composite_ids, all_ids) + def generate_top_comments(ncid=15, ncomm=100, minchar=50, maxchar=500): claim_ids = [fake.sha1() for _ in range(ncid)] From 0f5ea0f88fcc49330863df1988423626f4c26f99 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 18:05:03 -0400 Subject: [PATCH 10/21] Fixes importerror --- src/__init__.py | 2 -- src/database/__init__.py | 0 src/server/app.py | 4 ++-- 3 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 src/database/__init__.py diff --git a/src/__init__.py b/src/__init__.py index b28b04f..8b13789 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,3 +1 @@ - - diff --git a/src/database/__init__.py b/src/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server/app.py b/src/server/app.py index 2512801..520120f 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -9,8 +9,8 @@ import aiojobs.aiohttp import asyncio from aiohttp import web -from database.queries import setup_database, backup_database -from database.queries import obtain_connection, DatabaseWriter +from src.database.queries import setup_database, backup_database +from src.database.queries import obtain_connection, DatabaseWriter from src.server.handles import api_endpoint, get_api_endpoint logger = logging.getLogger(__name__) From f898173d9dffdb5551a28c7dae7f53bafb3a4100 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 18:06:13 -0400 Subject: [PATCH 11/21] Adds hide comment function to daemon --- src/database/writes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/database/writes.py b/src/database/writes.py index df0d950..3e1bf67 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -4,13 +4,12 @@ import sqlite3 from asyncio import coroutine from src.database.queries import delete_comment_by_id -from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim - from src.database.queries import get_comment_or_none from src.database.queries import insert_comment from src.database.queries import insert_channel from src.database.queries import get_channel_id_from_comment_id from src.database.queries import hide_comment_by_id +from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim from src.server.misc import channel_matches_pattern_or_error logger = logging.getLogger(__name__) From d5dfd5a53b11fb2a27e7ce56ad848f7b479621aa Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 18:07:21 -0400 Subject: [PATCH 12/21] Revert "Adds hide comment function to daemon" This reverts commit f898173d --- src/database/writes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/database/writes.py b/src/database/writes.py index 3e1bf67..df0d950 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -4,12 +4,13 @@ import sqlite3 from asyncio import coroutine from src.database.queries import delete_comment_by_id +from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim + from src.database.queries import get_comment_or_none from src.database.queries import insert_comment from src.database.queries import insert_channel from src.database.queries import get_channel_id_from_comment_id from src.database.queries import hide_comment_by_id -from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim from src.server.misc import channel_matches_pattern_or_error logger = logging.getLogger(__name__) From 28944ad9b346c37f1cb70805ccaafb91d843c4cc Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 18:09:14 -0400 Subject: [PATCH 13/21] Fixes ImportError --- src/database/writes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/database/writes.py b/src/database/writes.py index df0d950..a1cc57b 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -4,13 +4,14 @@ import sqlite3 from asyncio import coroutine from src.database.queries import delete_comment_by_id -from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim - from src.database.queries import get_comment_or_none from src.database.queries import insert_comment from src.database.queries import insert_channel from src.database.queries import get_channel_id_from_comment_id from src.database.queries import hide_comment_by_id +from src.server.misc import is_authentic_delete_signal +from src.server.misc import request_lbrynet +from src.server.misc import validate_signature_from_claim from src.server.misc import channel_matches_pattern_or_error logger = logging.getLogger(__name__) From ffb3711ac920894bda54ccc01f2cbfee9738a9aa Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Sun, 4 Aug 2019 18:21:07 -0400 Subject: [PATCH 14/21] get_hidden_claim_comments -> get_claim_hidden_comments --- src/database/queries.py | 2 +- src/server/handles.py | 8 ++++---- tests/database_test.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/database/queries.py b/src/database/queries.py index 7dd780c..4d0cf90 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -73,7 +73,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = } -def get_hidden_claim_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50): +def get_claim_hidden_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50): with conn: results = conn.execute( SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden = ? LIMIT ? OFFSET ?", diff --git a/src/server/handles.py b/src/server/handles.py index 662e13b..5951960 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -10,7 +10,7 @@ from src.server.misc import clean_input_params from src.database.queries import get_claim_comments from src.database.queries import get_comments_by_id, get_comment_ids from src.database.queries import get_channel_id_from_comment_id -from src.database.queries import get_hidden_claim_comments +from src.database.queries import get_claim_hidden_comments from src.server.misc import is_valid_base_comment from src.server.misc import is_valid_credential_input from src.server.misc import make_error @@ -43,8 +43,8 @@ def handle_get_comments_by_id(app, kwargs): return get_comments_by_id(app['reader'], **kwargs) -def handle_get_hidden_claim_comments(app, kwargs): - return get_hidden_claim_comments(app['reader'], **kwargs) +def handle_get_claim_hidden_comments(app, kwargs): + return get_claim_hidden_comments(app['reader'], **kwargs) async def handle_create_comment(app, params): @@ -66,7 +66,7 @@ async def handle_hide_comment(app, params): METHODS = { 'ping': ping, 'get_claim_comments': handle_get_claim_comments, - 'get_hidden_claim_comments': handle_get_hidden_claim_comments, + 'get_claim_hidden_comments': handle_get_claim_hidden_comments, 'get_comment_ids': handle_get_comment_ids, 'get_comments_by_id': handle_get_comments_by_id, 'get_channel_from_comment_id': handle_get_channel_from_comment_id, diff --git a/tests/database_test.py b/tests/database_test.py index 02050a0..ebf3477 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -8,7 +8,7 @@ from faker.providers import misc from src.database.queries import get_comments_by_id from src.database.queries import get_comment_ids from src.database.queries import get_claim_comments -from src.database.queries import get_hidden_claim_comments +from src.database.queries import get_claim_hidden_comments from src.database.writes import create_comment_or_error from src.database.queries import hide_comment_by_id from src.database.queries import delete_comment_by_id @@ -263,17 +263,17 @@ class ListDatabaseTest(DatabaseTestCase): self.assertFalse(comment_list['has_hidden_comments']) hide_comment_by_id(self.conn, comm2['comment_id']) - default_comments = get_hidden_claim_comments(self.conn, claim_id) + default_comments = get_claim_hidden_comments(self.conn, claim_id) self.assertIn('has_hidden_comments', default_comments) - hidden_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=True) + hidden_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=True) self.assertIn('has_hidden_comments', hidden_comments) self.assertEqual(default_comments, hidden_comments) hidden_comment = hidden_comments['items'][0] self.assertEqual(hidden_comment['comment_id'], comm2['comment_id']) - visible_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=False) + visible_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=False) self.assertIn('has_hidden_comments', visible_comments) self.assertNotIn(hidden_comment, visible_comments['items']) From 6d2447bd294180af7c2da97976dff4bdfbc9d674 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 9 Aug 2019 00:48:55 -0400 Subject: [PATCH 15/21] Adds queries for hiding comments + updates other queries --- src/database/queries.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/database/queries.py b/src/database/queries.py index 4d0cf90..b4e2dd1 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -18,6 +18,12 @@ SELECT_COMMENTS_ON_CLAIMS = """ FROM COMMENTS_ON_CLAIMS """ +SELECT_COMMENTS_ON_CLAIMS_CLAIMID = """ + SELECT comment, comment_id, claim_id, channel_name, channel_id, channel_url, + timestamp, signature, signing_ts, parent_id, is_hidden + FROM COMMENTS_ON_CLAIMS +""" + def clean(thing: dict) -> dict: if 'is_hidden' in thing: @@ -123,7 +129,7 @@ def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict: with conn: - curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,)) + curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS_CLAIMID + "WHERE comment_id = ?", (comment_id,)) thing = curry.fetchone() return clean(dict(thing)) if thing else None @@ -152,13 +158,13 @@ def get_comment_ids(conn: sqlite3.Connection, claim_id: str, parent_id: str = No return [tuple(row)[0] for row in curs.fetchall()] -def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]: +def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.Union[list, None]: """ Returns a list containing the comment data associated with each ID within the list""" # format the input, under the assumption that the placeholders = ', '.join('?' for _ in comment_ids) with conn: return [clean(dict(row)) for row in conn.execute( - SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})', + SELECT_COMMENTS_ON_CLAIMS_CLAIMID + f'WHERE comment_id IN ({placeholders})', tuple(comment_ids) )] @@ -169,12 +175,6 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str): return bool(curs.rowcount) -def hide_comment_by_id(conn: sqlite3.Connection, comment_id: str): - with conn: - curs = conn.execute("UPDATE OR IGNORE COMMENT SET IsHidden = TRUE WHERE CommentId = ?", (comment_id,)) - return bool(curs.rowcount) - - def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str): with conn: curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', (channel_id, channel_name)) @@ -189,6 +189,26 @@ def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str): return dict(channel) if channel else {} +def get_claim_ids_from_comment_ids(conn: sqlite3.Connection, comment_ids: list): + with conn: + cids = conn.execute( + f""" SELECT CommentId as comment_id, LbryClaimId AS claim_id FROM COMMENT + WHERE CommentId IN ({', '.join('?' for _ in comment_ids)}) """, + tuple(comment_ids) + ) + return {row['comment_id']: row['claim_id'] for row in cids.fetchall()} + + +def hide_comments_by_id(conn: sqlite3.Connection, comment_ids: list): + with conn: + curs = conn.cursor() + curs.executemany( + "UPDATE COMMENT SET IsHidden = TRUE WHERE CommentId = ?", + [[c] for c in comment_ids] + ) + return bool(curs.rowcount) + + class DatabaseWriter(object): _writer = None From d64044d3b258cc980ef27b75b063520c8938bf98 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 9 Aug 2019 00:50:59 -0400 Subject: [PATCH 16/21] Adds claim_search wrapper for lbrynet api --- src/database/writes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/database/writes.py b/src/database/writes.py index a1cc57b..3be5475 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -57,6 +57,8 @@ async def write_comment(app, params): async def hide_comment(app, comment_id): return await coroutine(hide_comment_by_id)(app['writer'], comment_id) +async def claim_search(app, **kwargs): + return (await request_lbrynet(app, 'claim_search', **kwargs))['items'][0] # comment_ids: [ # { From dc01e9b251f1a7ad2798e48e41a567b2a67f9ad0 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 9 Aug 2019 00:51:56 -0400 Subject: [PATCH 17/21] Replace singular hide_comment function with batch hide_comments function --- src/database/writes.py | 48 +++++++++++++++++++++++------------------- src/server/handles.py | 8 +++---- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/database/writes.py b/src/database/writes.py index 3be5475..16d1f17 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -3,12 +3,12 @@ import sqlite3 from asyncio import coroutine +from src.database.queries import hide_comments_by_id from src.database.queries import delete_comment_by_id from src.database.queries import get_comment_or_none from src.database.queries import insert_comment from src.database.queries import insert_channel -from src.database.queries import get_channel_id_from_comment_id -from src.database.queries import hide_comment_by_id +from src.database.queries import get_claim_ids_from_comment_ids from src.server.misc import is_authentic_delete_signal from src.server.misc import request_lbrynet from src.server.misc import validate_signature_from_claim @@ -54,28 +54,32 @@ async def write_comment(app, params): return await coroutine(create_comment_or_error)(app['writer'], **params) -async def hide_comment(app, comment_id): - return await coroutine(hide_comment_by_id)(app['writer'], comment_id) +async def hide_comments(app, comment_ids): + return await coroutine(hide_comments_by_id)(app['writer'], comment_ids) + async def claim_search(app, **kwargs): return (await request_lbrynet(app, 'claim_search', **kwargs))['items'][0] -# comment_ids: [ -# { -# "comment_id": id, -# "signing_ts": signing_ts, -# "signature": signature -# }, -# ... -# ] -async def hide_comment_if_authorized(app, comment_id, signing_ts, signature): - channel = get_channel_id_from_comment_id(app['reader'], comment_id) - claim = await request_lbrynet(app, 'claim_search', claim_id=channel['channel_id']) - claim = claim['items'][0] - if not validate_signature_from_claim(claim, signature, signing_ts, comment_id): - raise ValueError('Invalid Signature') - job = await app['comment_scheduler'].spawn(hide_comment(app, comment_id)) - return { - 'hidden': await job.wait() - } +async def hide_comments_where_authorized(app, pieces: list): + comment_cids = get_claim_ids_from_comment_ids( + conn=app['reader'], + comment_ids=[p['comment_id'] for p in pieces] + ) + # TODO: Amortize this process + claims = {} + comments_to_hide = [] + for p in pieces: + claim_id = comment_cids[p['comment_id']] + if claim_id not in claims: + claims[claim_id] = await claim_search(app, claim_id=claim_id, no_totals=True) + channel = claims[claim_id].get('signing_channel') + if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']): + comments_to_hide.append(p['comment_id']) + + if comments_to_hide: + job = await app['comment_scheduler'].spawn(hide_comments(app, comments_to_hide)) + await job.wait() + + return {'hidden': comments_to_hide} diff --git a/src/server/handles.py b/src/server/handles.py index 5951960..1f6cc44 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -16,7 +16,7 @@ from src.server.misc import is_valid_credential_input from src.server.misc import make_error from src.database.writes import delete_comment_if_authorized from src.database.writes import write_comment -from src.database.writes import hide_comment_if_authorized +from src.database.writes import hide_comments_where_authorized logger = logging.getLogger(__name__) @@ -59,8 +59,8 @@ async def handle_delete_comment(app, params): return await delete_comment_if_authorized(app, **params) -async def handle_hide_comment(app, params): - return await hide_comment_if_authorized(app, **params) +async def handle_hide_comments(app, params): + return await hide_comments_where_authorized(app, **params) METHODS = { @@ -73,7 +73,7 @@ METHODS = { 'create_comment': handle_create_comment, 'delete_comment': handle_delete_comment, 'abandon_comment': handle_delete_comment, - 'hide_comment': handle_hide_comment, + 'hide_comments': handle_hide_comments } From 3e7af6ee47e3efb525c39f9299a028fdcaa397e0 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 9 Aug 2019 00:52:34 -0400 Subject: [PATCH 18/21] Adds script to count valid signatures in the database --- scripts/valid_signatures.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 scripts/valid_signatures.py diff --git a/scripts/valid_signatures.py b/scripts/valid_signatures.py new file mode 100644 index 0000000..cc206d8 --- /dev/null +++ b/scripts/valid_signatures.py @@ -0,0 +1,84 @@ +import binascii +import hashlib +import json +import sqlite3 +import asyncio + +import aiohttp + +from src.server.misc import is_signature_valid, get_encoded_signature +from src.server.database import clean + + +async def request_lbrynet(url, method, **params): + body = {'method': method, 'params': {**params}} + async with aiohttp.request('POST', url, json=body) as req: + try: + resp = await req.json() + finally: + if 'result' in resp: + return resp['result'] + + +def get_comments_with_signatures(_conn: sqlite3.Connection) -> list: + with _conn: + curs = _conn.execute("SELECT * FROM COMMENTS_ON_CLAIMS WHERE signature IS NOT NULL") + return [dict(r) for r in curs.fetchall()] + + +def is_valid_signature(pubkey, channel_id, signature, signing_ts, data: str) -> bool: + try: + if pubkey: + claim_hash = binascii.unhexlify(channel_id.encode())[::-1] + injest = b''.join((signing_ts.encode(), claim_hash, data.encode())) + return is_signature_valid( + encoded_signature=get_encoded_signature(signature), + signature_digest=hashlib.sha256(injest).digest(), + public_key_bytes=binascii.unhexlify(pubkey.encode()) + ) + else: + raise Exception("Pubkey is null") + except Exception as e: + print(e) + return False + + +async def get_channel_pubkeys(comments: list): + urls = {c['channel_url'] for c in comments} + claims = await request_lbrynet('http://localhost:5279', 'resolve', urls=list(urls)) + cids = {c['channel_id']: None for c in comments} + error_claims = [] + for url, claim in claims.items(): + if 'error' not in claim: + cids.update({ + claim['claim_id']: claim['value']['public_key'] + }) + else: + error_claims.append({url: claim}) + return cids, error_claims + + +def count_valid_signatures(cmts: list, chan_pubkeys: dict): + invalid_comments = [] + for c in cmts: + pubkey = chan_pubkeys.get(c['channel_id']) + if not is_valid_signature(pubkey, c['channel_id'], c['signature'], c['signing_ts'], c['comment']): + invalid_comments.append(c) + return len(cmts) - len(invalid_comments), invalid_comments + + +if __name__ == '__main__': + conn = sqlite3.connect('database/default.db') + conn.row_factory = sqlite3.Row + comments = get_comments_with_signatures(conn) + loop = asyncio.get_event_loop() + chan_keys, errored = loop.run_until_complete(get_channel_pubkeys(comments)) + valid_sigs, invalid_coms = count_valid_signatures(comments, chan_keys) + print(f'Total Signatures: {len(comments)}\nValid Signatures: {valid_sigs}') + print(f'Invalid Signatures: {len(comments) - valid_sigs}') + print(f'Percent Valid: {round(valid_sigs/len(comments)*100, 3)}%') + print(f'# Unresolving claims: {len(errored)}') + print(f'Num invalid comments: {len(invalid_coms)}') + print(json.dumps(errored, indent=2)) + json.dump(invalid_coms, 'invalid_coms.json', indent=2) + From 7f9e475324e817e1f8e6738b00c79b9be05872e1 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 9 Aug 2019 00:53:10 -0400 Subject: [PATCH 19/21] `request.forwarded` -> `request.remote` --- src/server/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/handles.py b/src/server/handles.py index 1f6cc44..77ea225 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -108,7 +108,7 @@ async def process_json(app, body: dict) -> dict: @atomic async def api_endpoint(request: web.Request): try: - web.access_logger.info(f'Forwarded headers: {request.forwarded}') + web.access_logger.info(f'Forwarded headers: {request.remote}') body = await request.json() if type(body) is list or type(body) is dict: if type(body) is list: From 332fa2bcd068157d3c01540db52d1a31025023a7 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 9 Aug 2019 01:20:00 -0400 Subject: [PATCH 20/21] Updates unittests to be compliant with batched hide comments refactor --- tests/database_test.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/database_test.py b/tests/database_test.py index ebf3477..adf0e4a 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -10,7 +10,7 @@ from src.database.queries import get_comment_ids from src.database.queries import get_claim_comments from src.database.queries import get_claim_hidden_comments from src.database.writes import create_comment_or_error -from src.database.queries import hide_comment_by_id +from src.database.queries import hide_comments_by_id from src.database.queries import delete_comment_by_id from tests.testcase import DatabaseTestCase @@ -199,11 +199,11 @@ class TestDatabaseOperations(DatabaseTestCase): comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() self.assertFalse(comment['is_hidden']) - success = hide_comment_by_id(self.conn, comm['comment_id']) + success = hide_comments_by_id(self.conn, [comm['comment_id']]) self.assertTrue(success) comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() self.assertTrue(comment['is_hidden']) - success = hide_comment_by_id(self.conn, comm['comment_id']) + success = hide_comments_by_id(self.conn, [comm['comment_id']]) self.assertTrue(success) comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() self.assertTrue(comment['is_hidden']) @@ -211,11 +211,13 @@ class TestDatabaseOperations(DatabaseTestCase): def test08DeleteComments(self): comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') comments = get_claim_comments(self.conn, self.claimId) - self.assertIn(comm, comments['items']) + match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items'])) + self.assertTrue(match) deleted = delete_comment_by_id(self.conn, comm['comment_id']) self.assertTrue(deleted) comments = get_claim_comments(self.conn, self.claimId) - self.assertNotIn(comm, comments['items']) + match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items'])) + self.assertFalse(match) deleted = delete_comment_by_id(self.conn, comm['comment_id']) self.assertFalse(deleted) @@ -261,7 +263,7 @@ class ListDatabaseTest(DatabaseTestCase): self.assertEqual(len(comments), comment_list['total_items']) self.assertIn('has_hidden_comments', comment_list) self.assertFalse(comment_list['has_hidden_comments']) - hide_comment_by_id(self.conn, comm2['comment_id']) + hide_comments_by_id(self.conn, [comm2['comment_id']]) default_comments = get_claim_hidden_comments(self.conn, claim_id) self.assertIn('has_hidden_comments', default_comments) From 6933a9110d3694e39ab141159fb5c24a472bfec8 Mon Sep 17 00:00:00 2001 From: Oleg Silkin Date: Fri, 9 Aug 2019 03:12:54 -0400 Subject: [PATCH 21/21] update --- config/conf.json | 2 +- tests/database_test.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/config/conf.json b/config/conf.json index a41249f..4e4d027 100644 --- a/config/conf.json +++ b/config/conf.json @@ -1,6 +1,6 @@ { "PATH": { - "DATABASE": "database/comments.db", + "DATABASE": "database/default.db", "ERROR_LOG": "logs/error.log", "DEBUG_LOG": "logs/debug.log", "SERVER_LOG": "logs/server.log" diff --git a/tests/database_test.py b/tests/database_test.py index adf0e4a..068e45e 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -222,7 +222,6 @@ class TestDatabaseOperations(DatabaseTestCase): self.assertFalse(deleted) - class ListDatabaseTest(DatabaseTestCase): def setUp(self) -> None: super().setUp()