diff --git a/config/conf.json b/config/conf.json index a41249f..4e4d027 100644 --- a/config/conf.json +++ b/config/conf.json @@ -1,6 +1,6 @@ { "PATH": { - "DATABASE": "database/comments.db", + "DATABASE": "database/default.db", "ERROR_LOG": "logs/error.log", "DEBUG_LOG": "logs/debug.log", "SERVER_LOG": "logs/server.log" diff --git a/scripts/valid_signatures.py b/scripts/valid_signatures.py new file mode 100644 index 0000000..cc206d8 --- /dev/null +++ b/scripts/valid_signatures.py @@ -0,0 +1,84 @@ +import binascii +import hashlib +import json +import sqlite3 +import asyncio + +import aiohttp + +from src.server.misc import is_signature_valid, get_encoded_signature +from src.server.database import clean + + +async def request_lbrynet(url, method, **params): + body = {'method': method, 'params': {**params}} + async with aiohttp.request('POST', url, json=body) as req: + try: + resp = await req.json() + finally: + if 'result' in resp: + return resp['result'] + + +def get_comments_with_signatures(_conn: sqlite3.Connection) -> list: + with _conn: + curs = _conn.execute("SELECT * FROM COMMENTS_ON_CLAIMS WHERE signature IS NOT NULL") + return [dict(r) for r in curs.fetchall()] + + +def is_valid_signature(pubkey, channel_id, signature, signing_ts, data: str) -> bool: + try: + if pubkey: + claim_hash = binascii.unhexlify(channel_id.encode())[::-1] + injest = b''.join((signing_ts.encode(), claim_hash, data.encode())) + return is_signature_valid( + encoded_signature=get_encoded_signature(signature), + signature_digest=hashlib.sha256(injest).digest(), + public_key_bytes=binascii.unhexlify(pubkey.encode()) + ) + else: + raise Exception("Pubkey is null") + except Exception as e: + print(e) + return False + + +async def get_channel_pubkeys(comments: list): + urls = {c['channel_url'] for c in comments} + claims = await request_lbrynet('http://localhost:5279', 'resolve', urls=list(urls)) + cids = {c['channel_id']: None for c in comments} + error_claims = [] + for url, claim in claims.items(): + if 'error' not in claim: + cids.update({ + claim['claim_id']: claim['value']['public_key'] + }) + else: + error_claims.append({url: claim}) + return cids, error_claims + + +def count_valid_signatures(cmts: list, chan_pubkeys: dict): + invalid_comments = [] + for c in cmts: + pubkey = chan_pubkeys.get(c['channel_id']) + if not is_valid_signature(pubkey, c['channel_id'], c['signature'], c['signing_ts'], c['comment']): + invalid_comments.append(c) + return len(cmts) - len(invalid_comments), invalid_comments + + +if __name__ == '__main__': + conn = sqlite3.connect('database/default.db') + conn.row_factory = sqlite3.Row + comments = get_comments_with_signatures(conn) + loop = asyncio.get_event_loop() + chan_keys, errored = loop.run_until_complete(get_channel_pubkeys(comments)) + valid_sigs, invalid_coms = count_valid_signatures(comments, chan_keys) + print(f'Total Signatures: {len(comments)}\nValid Signatures: {valid_sigs}') + print(f'Invalid Signatures: {len(comments) - valid_sigs}') + print(f'Percent Valid: {round(valid_sigs/len(comments)*100, 3)}%') + print(f'# Unresolving claims: {len(errored)}') + print(f'Num invalid comments: {len(invalid_coms)}') + print(json.dumps(errored, indent=2)) + json.dump(invalid_coms, 'invalid_coms.json', indent=2) + diff --git a/src/__init__.py b/src/__init__.py index b28b04f..8b13789 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,3 +1 @@ - - diff --git a/src/database/__init__.py b/src/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/database/comments_ddl.sql b/src/database/comments_ddl.sql index 7556fbd..fc85acf 100644 --- a/src/database/comments_ddl.sql +++ b/src/database/comments_ddl.sql @@ -9,12 +9,13 @@ CREATE TABLE IF NOT EXISTS COMMENT ( CommentId TEXT NOT NULL, LbryClaimId TEXT NOT NULL, - ChannelId TEXT DEFAULT NULL, + ChannelId TEXT DEFAULT (NULL), Body TEXT NOT NULL, - ParentId TEXT DEFAULT NULL, - Signature TEXT DEFAULT NULL, + ParentId TEXT DEFAULT (NULL), + Signature TEXT DEFAULT (NULL), Timestamp INTEGER NOT NULL, - SigningTs TEXT DEFAULT NULL, + SigningTs TEXT DEFAULT (NULL), + IsHidden BOOLEAN NOT NULL DEFAULT (FALSE), CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) @@ -23,6 +24,7 @@ CREATE TABLE IF NOT EXISTS COMMENT ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level ); +-- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE); -- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL; -- DROP TABLE IF EXISTS CHANNEL; @@ -37,27 +39,26 @@ CREATE TABLE IF NOT EXISTS CHANNEL -- indexes -- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX; -CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); +-- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); -CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); +-- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); -- VIEWS -CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS (comment_id, claim_id, timestamp, channel_name, channel_id, channel_url, - signature, signing_ts, parent_id, comment) AS -SELECT C.CommentId, - C.LbryClaimId, - C.Timestamp, - CHAN.Name, - CHAN.ClaimId, - 'lbry://' || CHAN.Name || '#' || CHAN.ClaimId, - C.Signature, - C.SigningTs, - C.ParentId, - C.Body -FROM COMMENT AS C - LEFT OUTER JOIN CHANNEL CHAN on C.ChannelId = CHAN.ClaimId -ORDER BY C.Timestamp DESC; - +CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT + C.CommentId AS comment_id, + C.Body AS comment, + C.LbryClaimId AS claim_id, + C.Timestamp AS timestamp, + CHAN.Name AS channel_name, + CHAN.ClaimId AS channel_id, + ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, + C.Signature AS signature, + C.SigningTs AS signing_ts, + C.ParentId AS parent_id, + C.IsHidden AS is_hidden + FROM COMMENT AS C + LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId + ORDER BY C.Timestamp DESC; DROP VIEW IF EXISTS COMMENT_REPLIES; diff --git a/src/database/queries.py b/src/database/queries.py index 417cf15..b4e2dd1 100644 --- a/src/database/queries.py +++ b/src/database/queries.py @@ -12,8 +12,23 @@ from src.database.schema import CREATE_TABLES_QUERY logger = logging.getLogger(__name__) +SELECT_COMMENTS_ON_CLAIMS = """ + SELECT comment, comment_id, channel_name, channel_id, channel_url, + timestamp, signature, signing_ts, parent_id, is_hidden + FROM COMMENTS_ON_CLAIMS +""" + +SELECT_COMMENTS_ON_CLAIMS_CLAIMID = """ + SELECT comment, comment_id, claim_id, channel_name, channel_id, channel_url, + timestamp, signature, signing_ts, parent_id, is_hidden + FROM COMMENTS_ON_CLAIMS +""" + + def clean(thing: dict) -> dict: - return {k: v for k, v in thing.items() if v} + if 'is_hidden' in thing: + thing.update({'is_hidden': bool(thing['is_hidden'])}) + return {k: v for k, v in thing.items() if v is not None} def obtain_connection(filepath: str = None, row_factory: bool = True): @@ -28,51 +43,30 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = with conn: if top_level: results = [clean(dict(row)) for row in conn.execute( - """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id IS NULL - LIMIT ? OFFSET ? """, + SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?", (claim_id, page_size, page_size * (page - 1)) )] count = conn.execute( - """ - SELECT COUNT(*) - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id IS NULL - """, (claim_id,) + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id IS NULL", + (claim_id,) ) elif parent_id is None: results = [clean(dict(row)) for row in conn.execute( - """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? - LIMIT ? OFFSET ? """, + SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ", (claim_id, page_size, page_size * (page - 1)) )] count = conn.execute( - """ - SELECT COUNT(*) - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? - """, (claim_id,) + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ?", + (claim_id,) ) else: results = [clean(dict(row)) for row in conn.execute( - """ SELECT comment, comment_id, channel_name, channel_id, - channel_url, timestamp, signature, signing_ts, parent_id - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id = ? - LIMIT ? OFFSET ? """, + SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? ", (claim_id, parent_id, page_size, page_size * (page - 1)) )] count = conn.execute( - """ - SELECT COUNT(*) - FROM COMMENTS_ON_CLAIMS - WHERE claim_id = ? AND parent_id = ? - """, (claim_id, parent_id) + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id = ?", + (claim_id, parent_id) ) count = tuple(count.fetchone())[0] return { @@ -80,10 +74,42 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str = 'page': page, 'page_size': page_size, 'total_pages': math.ceil(count / page_size), - 'total_items': count + 'total_items': count, + 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id) } +def get_claim_hidden_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50): + with conn: + results = conn.execute( + SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden = ? LIMIT ? OFFSET ?", + (claim_id, hidden, page_size, page_size * (page - 1)) + ) + count = conn.execute( + "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = ?", (claim_id, hidden) + ) + results = [clean(dict(row)) for row in results.fetchall()] + count = tuple(count.fetchone())[0] + + return { + 'items': results, + 'page': page, + 'page_size': page_size, + 'total_pages': math.ceil(count/page_size), + 'total_items': count, + 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id) + } + + +def claim_has_hidden_comments(conn, claim_id): + with conn: + result = conn.execute( + "SELECT COUNT(DISTINCT is_hidden) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = TRUE", + (claim_id,) + ) + return bool(tuple(result.fetchone())[0]) + + def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent_id: str = None, channel_id: str = None, signature: str = None, signing_ts: str = None) -> str: timestamp = int(time.time()) @@ -103,13 +129,7 @@ def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict: with conn: - curry = conn.execute( - """ - SELECT comment, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id - FROM COMMENTS_ON_CLAIMS WHERE comment_id = ? - """, - (comment_id,) - ) + curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS_CLAIMID + "WHERE comment_id = ?", (comment_id,)) thing = curry.fetchone() return clean(dict(thing)) if thing else None @@ -138,26 +158,17 @@ def get_comment_ids(conn: sqlite3.Connection, claim_id: str, parent_id: str = No return [tuple(row)[0] for row in curs.fetchall()] -def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]: +def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.Union[list, None]: """ Returns a list containing the comment data associated with each ID within the list""" # format the input, under the assumption that the placeholders = ', '.join('?' for _ in comment_ids) with conn: return [clean(dict(row)) for row in conn.execute( - f'SELECT * FROM COMMENTS_ON_CLAIMS WHERE comment_id IN ({placeholders})', + SELECT_COMMENTS_ON_CLAIMS_CLAIMID + f'WHERE comment_id IN ({placeholders})', tuple(comment_ids) )] -def delete_anonymous_comment_by_id(conn: sqlite3.Connection, comment_id: str): - with conn: - curs = conn.execute( - "DELETE FROM COMMENT WHERE ChannelId IS NULL AND CommentId = ?", - (comment_id,) - ) - return curs.rowcount - - def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str): with conn: curs = conn.execute("DELETE FROM COMMENT WHERE CommentId = ?", (comment_id,)) @@ -166,19 +177,36 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str): def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str): with conn: - conn.execute( - 'INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', - (channel_id, channel_name) - ) + curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', (channel_id, channel_name)) + return bool(curs.rowcount) def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str): with conn: - channel = conn.execute(""" - SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ? - """, (comment_id,) + channel = conn.execute( + "SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?", (comment_id,) ).fetchone() - return dict(channel) if channel else dict() + return dict(channel) if channel else {} + + +def get_claim_ids_from_comment_ids(conn: sqlite3.Connection, comment_ids: list): + with conn: + cids = conn.execute( + f""" SELECT CommentId as comment_id, LbryClaimId AS claim_id FROM COMMENT + WHERE CommentId IN ({', '.join('?' for _ in comment_ids)}) """, + tuple(comment_ids) + ) + return {row['comment_id']: row['claim_id'] for row in cids.fetchall()} + + +def hide_comments_by_id(conn: sqlite3.Connection, comment_ids: list): + with conn: + curs = conn.cursor() + curs.executemany( + "UPDATE COMMENT SET IsHidden = TRUE WHERE CommentId = ?", + [[c] for c in comment_ids] + ) + return bool(curs.rowcount) class DatabaseWriter(object): diff --git a/src/database/schema.py b/src/database/schema.py index bf46f72..f474765 100644 --- a/src/database/schema.py +++ b/src/database/schema.py @@ -6,12 +6,13 @@ CREATE_COMMENT_TABLE = """ CREATE TABLE IF NOT EXISTS COMMENT ( CommentId TEXT NOT NULL, LbryClaimId TEXT NOT NULL, - ChannelId TEXT DEFAULT NULL, + ChannelId TEXT DEFAULT NULL, Body TEXT NOT NULL, - ParentId TEXT DEFAULT NULL, - Signature TEXT DEFAULT NULL, + ParentId TEXT DEFAULT NULL, + Signature TEXT DEFAULT NULL, Timestamp INTEGER NOT NULL, - SigningTs TEXT DEFAULT NULL, + SigningTs TEXT DEFAULT NULL, + IsHidden BOOLEAN NOT NULL DEFAULT (FALSE), CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) @@ -46,7 +47,8 @@ CREATE_COMMENTS_ON_CLAIMS_VIEW = """ ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, C.Signature AS signature, C.SigningTs AS signing_ts, - C.ParentId AS parent_id + C.ParentId AS parent_id, + C.IsHidden as is_hidden FROM COMMENT AS C LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId ORDER BY C.Timestamp DESC; diff --git a/src/database/writes.py b/src/database/writes.py index 0c4ba03..16d1f17 100644 --- a/src/database/writes.py +++ b/src/database/writes.py @@ -3,12 +3,15 @@ import sqlite3 from asyncio import coroutine -from database.queries import delete_comment_by_id +from src.database.queries import hide_comments_by_id +from src.database.queries import delete_comment_by_id +from src.database.queries import get_comment_or_none +from src.database.queries import insert_comment +from src.database.queries import insert_channel +from src.database.queries import get_claim_ids_from_comment_ids from src.server.misc import is_authentic_delete_signal - -from database.queries import get_comment_or_none -from database.queries import insert_comment -from database.queries import insert_channel +from src.server.misc import request_lbrynet +from src.server.misc import validate_signature_from_claim from src.server.misc import channel_matches_pattern_or_error logger = logging.getLogger(__name__) @@ -47,5 +50,36 @@ async def delete_comment_if_authorized(app, comment_id, **kwargs): return {'deleted': await job.wait()} -async def write_comment(app, comment): - return await coroutine(create_comment_or_error)(app['writer'], **comment) +async def write_comment(app, params): + return await coroutine(create_comment_or_error)(app['writer'], **params) + + +async def hide_comments(app, comment_ids): + return await coroutine(hide_comments_by_id)(app['writer'], comment_ids) + + +async def claim_search(app, **kwargs): + return (await request_lbrynet(app, 'claim_search', **kwargs))['items'][0] + + +async def hide_comments_where_authorized(app, pieces: list): + comment_cids = get_claim_ids_from_comment_ids( + conn=app['reader'], + comment_ids=[p['comment_id'] for p in pieces] + ) + # TODO: Amortize this process + claims = {} + comments_to_hide = [] + for p in pieces: + claim_id = comment_cids[p['comment_id']] + if claim_id not in claims: + claims[claim_id] = await claim_search(app, claim_id=claim_id, no_totals=True) + channel = claims[claim_id].get('signing_channel') + if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']): + comments_to_hide.append(p['comment_id']) + + if comments_to_hide: + job = await app['comment_scheduler'].spawn(hide_comments(app, comments_to_hide)) + await job.wait() + + return {'hidden': comments_to_hide} diff --git a/src/server/app.py b/src/server/app.py index 2512801..520120f 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -9,8 +9,8 @@ import aiojobs.aiohttp import asyncio from aiohttp import web -from database.queries import setup_database, backup_database -from database.queries import obtain_connection, DatabaseWriter +from src.database.queries import setup_database, backup_database +from src.database.queries import obtain_connection, DatabaseWriter from src.server.handles import api_endpoint, get_api_endpoint logger = logging.getLogger(__name__) diff --git a/src/server/handles.py b/src/server/handles.py index f1d8f98..77ea225 100644 --- a/src/server/handles.py +++ b/src/server/handles.py @@ -7,13 +7,17 @@ from aiohttp import web from aiojobs.aiohttp import atomic from src.server.misc import clean_input_params -from database.queries import get_claim_comments -from database.queries import get_comments_by_id, get_comment_ids -from database.queries import get_channel_id_from_comment_id +from src.database.queries import get_claim_comments +from src.database.queries import get_comments_by_id, get_comment_ids +from src.database.queries import get_channel_id_from_comment_id +from src.database.queries import get_claim_hidden_comments from src.server.misc import is_valid_base_comment from src.server.misc import is_valid_credential_input from src.server.misc import make_error -from database.writes import delete_comment_if_authorized, write_comment +from src.database.writes import delete_comment_if_authorized +from src.database.writes import write_comment +from src.database.writes import hide_comments_where_authorized + logger = logging.getLogger(__name__) @@ -24,23 +28,23 @@ def ping(*args): def handle_get_channel_from_comment_id(app, kwargs: dict): - with app['reader'] as conn: - return get_channel_id_from_comment_id(conn, **kwargs) + return get_channel_id_from_comment_id(app['reader'], **kwargs) def handle_get_comment_ids(app, kwargs): - with app['reader'] as conn: - return get_comment_ids(conn, **kwargs) + return get_comment_ids(app['reader'], **kwargs) def handle_get_claim_comments(app, kwargs): - with app['reader'] as conn: - return get_claim_comments(conn, **kwargs) + return get_claim_comments(app['reader'], **kwargs) def handle_get_comments_by_id(app, kwargs): - with app['reader'] as conn: - return get_comments_by_id(conn, **kwargs) + return get_comments_by_id(app['reader'], **kwargs) + + +def handle_get_claim_hidden_comments(app, kwargs): + return get_claim_hidden_comments(app['reader'], **kwargs) async def handle_create_comment(app, params): @@ -55,15 +59,21 @@ async def handle_delete_comment(app, params): return await delete_comment_if_authorized(app, **params) +async def handle_hide_comments(app, params): + return await hide_comments_where_authorized(app, **params) + + METHODS = { 'ping': ping, 'get_claim_comments': handle_get_claim_comments, + 'get_claim_hidden_comments': handle_get_claim_hidden_comments, 'get_comment_ids': handle_get_comment_ids, 'get_comments_by_id': handle_get_comments_by_id, 'get_channel_from_comment_id': handle_get_channel_from_comment_id, 'create_comment': handle_create_comment, 'delete_comment': handle_delete_comment, - # 'abandon_comment': handle_delete_comment, + 'abandon_comment': handle_delete_comment, + 'hide_comments': handle_hide_comments } @@ -98,7 +108,7 @@ async def process_json(app, body: dict) -> dict: @atomic async def api_endpoint(request: web.Request): try: - web.access_logger.info(f'Forwarded headers: {request.forwarded}') + web.access_logger.info(f'Forwarded headers: {request.remote}') body = await request.json() if type(body) is list or type(body) is dict: if type(body) is list: diff --git a/src/server/misc.py b/src/server/misc.py index d0efd66..07567b9 100644 --- a/src/server/misc.py +++ b/src/server/misc.py @@ -41,20 +41,19 @@ def make_error(error, exc=None) -> dict: return body -async def resolve_channel_claim(app, channel_id, channel_name): - lbry_url = f'lbry://{channel_name}#{channel_id}' - resolve_body = {'method': 'resolve', 'params': {'urls': [lbry_url]}} +async def request_lbrynet(app, method, **params): + body = {'method': method, 'params': {**params}} try: - async with aiohttp.request('POST', app['config']['LBRYNET'], json=resolve_body) as req: + async with aiohttp.request('POST', app['config']['LBRYNET'], json=body) as req: try: resp = await req.json() except JSONDecodeError as jde: logger.exception(jde.msg) - raise Exception('JSON Decode Error in Claim Resolution') + raise Exception('JSON Decode Error In lbrynet request') finally: if 'result' in resp: - return resp['result'].get(lbry_url) - raise ValueError('claim resolution yields error', {'error': resp['error']}) + return resp['result'] + raise ValueError('LBRYNET Request Error', {'error': resp['error']}) except (ConnectionRefusedError, ClientConnectorError): logger.critical("Connection to the LBRYnet daemon failed, make sure it's running.") raise Exception("Server cannot verify delete signature") @@ -111,7 +110,8 @@ def is_valid_credential_input(channel_id=None, channel_name=None, signature=None async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id, signature, signing_ts): - claim = await resolve_channel_claim(app, channel_id, channel_name) + lbry_url = f'lbry://{channel_name}#{channel_id}' + claim = await request_lbrynet(app, 'resolve', urls=[lbry_url]) if claim: public_key = claim['value']['public_key'] claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1] @@ -124,6 +124,21 @@ async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id, return False +def validate_signature_from_claim(claim, signature, signing_ts, data: str): + try: + if claim: + public_key = claim['value']['public_key'] + claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1] + injest = b''.join((signing_ts.encode(), claim_hash, data.encode())) + return is_signature_valid( + encoded_signature=get_encoded_signature(signature), + signature_digest=hashlib.sha256(injest).digest(), + public_key_bytes=binascii.unhexlify(public_key.encode()) + ) + except: + return False + + def clean_input_params(kwargs: dict): for k, v in kwargs.items(): if type(v) is str: diff --git a/tests/database_test.py b/tests/database_test.py index 3de9312..068e45e 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -5,10 +5,13 @@ from faker.providers import internet from faker.providers import lorem from faker.providers import misc -from database.queries import get_comments_by_id -from database.queries import get_comment_ids -from database.queries import get_claim_comments -from database.writes import create_comment_or_error +from src.database.queries import get_comments_by_id +from src.database.queries import get_comment_ids +from src.database.queries import get_claim_comments +from src.database.queries import get_claim_hidden_comments +from src.database.writes import create_comment_or_error +from src.database.queries import hide_comments_by_id +from src.database.queries import delete_comment_by_id from tests.testcase import DatabaseTestCase fake = faker.Faker() @@ -17,7 +20,7 @@ fake.add_provider(lorem) fake.add_provider(misc) -class TestCommentCreation(DatabaseTestCase): +class TestDatabaseOperations(DatabaseTestCase): def setUp(self) -> None: super().setUp() self.claimId = '529357c3422c6046d3fec76be2358004ba22e340' @@ -192,6 +195,32 @@ class TestCommentCreation(DatabaseTestCase): self.assertLessEqual(len(replies), 50) self.assertEqual(len(replies), len(comments_ids)) + def test07HideComments(self): + comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') + comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + self.assertFalse(comment['is_hidden']) + success = hide_comments_by_id(self.conn, [comm['comment_id']]) + self.assertTrue(success) + comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + self.assertTrue(comment['is_hidden']) + success = hide_comments_by_id(self.conn, [comm['comment_id']]) + self.assertTrue(success) + comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop() + self.assertTrue(comment['is_hidden']) + + def test08DeleteComments(self): + comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123') + comments = get_claim_comments(self.conn, self.claimId) + match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items'])) + self.assertTrue(match) + deleted = delete_comment_by_id(self.conn, comm['comment_id']) + self.assertTrue(deleted) + comments = get_claim_comments(self.conn, self.claimId) + match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items'])) + self.assertFalse(match) + deleted = delete_comment_by_id(self.conn, comm['comment_id']) + self.assertFalse(deleted) + class ListDatabaseTest(DatabaseTestCase): def setUp(self) -> None: @@ -204,6 +233,8 @@ class ListDatabaseTest(DatabaseTestCase): comments = get_claim_comments(self.conn, claim_id) self.assertIsNotNone(comments) self.assertGreater(comments['page_size'], 0) + self.assertIn('has_hidden_comments', comments) + self.assertFalse(comments['has_hidden_comments']) top_comments = get_claim_comments(self.conn, claim_id, top_level=True, page=1, page_size=50) self.assertIsNotNone(top_comments) self.assertEqual(top_comments['page_size'], 50) @@ -218,6 +249,45 @@ class ListDatabaseTest(DatabaseTestCase): self.assertIsNotNone(matching_comments) self.assertEqual(len(matching_comments), len(comment_ids)) + def testHiddenCommentLists(self): + claim_id = 'a'*40 + comm1 = create_comment_or_error(self.conn, 'Comment #1', claim_id, '1'*40, '@Doge123', 'a'*128, '123') + comm2 = create_comment_or_error(self.conn, 'Comment #2', claim_id, '1'*40, '@Doge123', 'b'*128, '123') + comm3 = create_comment_or_error(self.conn, 'Comment #3', claim_id, '1'*40, '@Doge123', 'c'*128, '123') + comments = [comm1, comm2, comm3] + + comment_list = get_claim_comments(self.conn, claim_id) + self.assertIn('items', comment_list) + self.assertIn('has_hidden_comments', comment_list) + self.assertEqual(len(comments), comment_list['total_items']) + self.assertIn('has_hidden_comments', comment_list) + self.assertFalse(comment_list['has_hidden_comments']) + hide_comments_by_id(self.conn, [comm2['comment_id']]) + + default_comments = get_claim_hidden_comments(self.conn, claim_id) + self.assertIn('has_hidden_comments', default_comments) + + hidden_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=True) + self.assertIn('has_hidden_comments', hidden_comments) + self.assertEqual(default_comments, hidden_comments) + + hidden_comment = hidden_comments['items'][0] + self.assertEqual(hidden_comment['comment_id'], comm2['comment_id']) + + visible_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=False) + self.assertIn('has_hidden_comments', visible_comments) + self.assertNotIn(hidden_comment, visible_comments['items']) + + hidden_ids = [c['comment_id'] for c in hidden_comments['items']] + visible_ids = [c['comment_id'] for c in visible_comments['items']] + composite_ids = hidden_ids + visible_ids + composite_ids.sort() + + comment_list = get_claim_comments(self.conn, claim_id) + all_ids = [c['comment_id'] for c in comment_list['items']] + all_ids.sort() + self.assertEqual(composite_ids, all_ids) + def generate_top_comments(ncid=15, ncomm=100, minchar=50, maxchar=500): claim_ids = [fake.sha1() for _ in range(ncid)] diff --git a/tests/testcase.py b/tests/testcase.py index 2357347..ddd8b44 100644 --- a/tests/testcase.py +++ b/tests/testcase.py @@ -6,7 +6,7 @@ from unittest.case import _Outcome import asyncio -from database.queries import obtain_connection, setup_database +from src.database.queries import obtain_connection, setup_database class AsyncioTestCase(unittest.TestCase):