LBRYCommunity/comment-server
1
0
Fork 0
Code Issues 18 Pull requests 1 Projects Releases Packages Wiki Activity

Merge pull request #7 from lbryio/hide-comments

Browse source 
Adds Hide Comment Functionality + Updates
\
This commit is contained in:
Oleg Silkin 2019-08-09 03:13:37 -04:00 committed by GitHub
commit d97dd5513d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 367 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/conf.json
View file

@ -1,6 +1,6 @@
{ {
"PATH": { "PATH": {
"DATABASE": "database/comments.db", "DATABASE": "database/default.db",
"ERROR_LOG": "logs/error.log", "ERROR_LOG": "logs/error.log",
"DEBUG_LOG": "logs/debug.log", "DEBUG_LOG": "logs/debug.log",
"SERVER_LOG": "logs/server.log" "SERVER_LOG": "logs/server.log"

84
scripts/valid_signatures.py Normal file
View file

@ -0,0 +1,84 @@
import binascii
import hashlib
import json
import sqlite3
import asyncio
import aiohttp
from src.server.misc import is_signature_valid, get_encoded_signature
from src.server.database import clean
async def request_lbrynet(url, method, **params):
body = {'method': method, 'params': {**params}}
async with aiohttp.request('POST', url, json=body) as req:
try:
resp = await req.json()
finally:
if 'result' in resp:
return resp['result']
def get_comments_with_signatures(_conn: sqlite3.Connection) -> list:
with _conn:
curs = _conn.execute("SELECT * FROM COMMENTS_ON_CLAIMS WHERE signature IS NOT NULL")
return [dict(r) for r in curs.fetchall()]
def is_valid_signature(pubkey, channel_id, signature, signing_ts, data: str) -> bool:
try:
if pubkey:
claim_hash = binascii.unhexlify(channel_id.encode())[::-1]
injest = b''.join((signing_ts.encode(), claim_hash, data.encode()))
return is_signature_valid(
encoded_signature=get_encoded_signature(signature),
signature_digest=hashlib.sha256(injest).digest(),
public_key_bytes=binascii.unhexlify(pubkey.encode())
)
else:
raise Exception("Pubkey is null")
except Exception as e:
print(e)
return False
async def get_channel_pubkeys(comments: list):
urls = {c['channel_url'] for c in comments}
claims = await request_lbrynet('http://localhost:5279', 'resolve', urls=list(urls))
cids = {c['channel_id']: None for c in comments}
error_claims = []
for url, claim in claims.items():
if 'error' not in claim:
cids.update({
claim['claim_id']: claim['value']['public_key']
})
else:
error_claims.append({url: claim})
return cids, error_claims
def count_valid_signatures(cmts: list, chan_pubkeys: dict):
invalid_comments = []
for c in cmts:
pubkey = chan_pubkeys.get(c['channel_id'])
if not is_valid_signature(pubkey, c['channel_id'], c['signature'], c['signing_ts'], c['comment']):
invalid_comments.append(c)
return len(cmts) - len(invalid_comments), invalid_comments
if __name__ == '__main__':
conn = sqlite3.connect('database/default.db')
conn.row_factory = sqlite3.Row
comments = get_comments_with_signatures(conn)
loop = asyncio.get_event_loop()
chan_keys, errored = loop.run_until_complete(get_channel_pubkeys(comments))
valid_sigs, invalid_coms = count_valid_signatures(comments, chan_keys)
print(f'Total Signatures: {len(comments)}\nValid Signatures: {valid_sigs}')
print(f'Invalid Signatures: {len(comments) - valid_sigs}')
print(f'Percent Valid: {round(valid_sigs/len(comments)*100, 3)}%')
print(f'# Unresolving claims: {len(errored)}')
print(f'Num invalid comments: {len(invalid_coms)}')
print(json.dumps(errored, indent=2))
json.dump(invalid_coms, 'invalid_coms.json', indent=2)

2
src/__init__.py
View file

@ -1,3 +1 @@

0
src/database/__init__.py Normal file
View file

45
src/database/comments_ddl.sql
View file

@ -9,12 +9,13 @@ CREATE TABLE IF NOT EXISTS COMMENT
( (
CommentId TEXT NOT NULL, CommentId TEXT NOT NULL,
LbryClaimId TEXT NOT NULL, LbryClaimId TEXT NOT NULL,
ChannelId TEXT DEFAULT NULL, ChannelId TEXT DEFAULT (NULL),
Body TEXT NOT NULL, Body TEXT NOT NULL,
ParentId TEXT DEFAULT NULL, ParentId TEXT DEFAULT (NULL),
Signature TEXT DEFAULT NULL, Signature TEXT DEFAULT (NULL),
Timestamp INTEGER NOT NULL, Timestamp INTEGER NOT NULL,
SigningTs TEXT DEFAULT NULL, SigningTs TEXT DEFAULT (NULL),
IsHidden BOOLEAN NOT NULL DEFAULT (FALSE),
CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
@ -23,6 +24,7 @@ CREATE TABLE IF NOT EXISTS COMMENT
ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
); );
-- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE);
-- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL; -- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL;
-- DROP TABLE IF EXISTS CHANNEL; -- DROP TABLE IF EXISTS CHANNEL;
@ -37,27 +39,26 @@ CREATE TABLE IF NOT EXISTS CHANNEL
-- indexes -- indexes
-- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX; -- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX;
CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId); -- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId); -- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
-- VIEWS -- VIEWS
CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS (comment_id, claim_id, timestamp, channel_name, channel_id, channel_url, CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT
signature, signing_ts, parent_id, comment) AS C.CommentId AS comment_id,
SELECT C.CommentId, C.Body AS comment,
C.LbryClaimId, C.LbryClaimId AS claim_id,
C.Timestamp, C.Timestamp AS timestamp,
CHAN.Name, CHAN.Name AS channel_name,
CHAN.ClaimId, CHAN.ClaimId AS channel_id,
'lbry://' || CHAN.Name || '#' || CHAN.ClaimId, ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
C.Signature, C.Signature AS signature,
C.SigningTs, C.SigningTs AS signing_ts,
C.ParentId, C.ParentId AS parent_id,
C.Body C.IsHidden AS is_hidden
FROM COMMENT AS C FROM COMMENT AS C
LEFT OUTER JOIN CHANNEL CHAN on C.ChannelId = CHAN.ClaimId LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
ORDER BY C.Timestamp DESC; ORDER BY C.Timestamp DESC;
DROP VIEW IF EXISTS COMMENT_REPLIES; DROP VIEW IF EXISTS COMMENT_REPLIES;

144
src/database/queries.py
View file

@ -12,8 +12,23 @@ from src.database.schema import CREATE_TABLES_QUERY
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
SELECT_COMMENTS_ON_CLAIMS = """
SELECT comment, comment_id, channel_name, channel_id, channel_url,
timestamp, signature, signing_ts, parent_id, is_hidden
FROM COMMENTS_ON_CLAIMS
"""
SELECT_COMMENTS_ON_CLAIMS_CLAIMID = """
SELECT comment, comment_id, claim_id, channel_name, channel_id, channel_url,
timestamp, signature, signing_ts, parent_id, is_hidden
FROM COMMENTS_ON_CLAIMS
"""
def clean(thing: dict) -> dict: def clean(thing: dict) -> dict:
return {k: v for k, v in thing.items() if v} if 'is_hidden' in thing:
thing.update({'is_hidden': bool(thing['is_hidden'])})
return {k: v for k, v in thing.items() if v is not None}
def obtain_connection(filepath: str = None, row_factory: bool = True): def obtain_connection(filepath: str = None, row_factory: bool = True):
@ -28,51 +43,30 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
with conn: with conn:
if top_level: if top_level:
results = [clean(dict(row)) for row in conn.execute( results = [clean(dict(row)) for row in conn.execute(
""" SELECT comment, comment_id, channel_name, channel_id, SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?",
channel_url, timestamp, signature, signing_ts, parent_id
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ? AND parent_id IS NULL
LIMIT ? OFFSET ? """,
(claim_id, page_size, page_size * (page - 1)) (claim_id, page_size, page_size * (page - 1))
)] )]
count = conn.execute( count = conn.execute(
""" "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id IS NULL",
SELECT COUNT(*) (claim_id,)
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ? AND parent_id IS NULL
""", (claim_id,)
) )
elif parent_id is None: elif parent_id is None:
results = [clean(dict(row)) for row in conn.execute( results = [clean(dict(row)) for row in conn.execute(
""" SELECT comment, comment_id, channel_name, channel_id, SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ",
channel_url, timestamp, signature, signing_ts, parent_id
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ?
LIMIT ? OFFSET ? """,
(claim_id, page_size, page_size * (page - 1)) (claim_id, page_size, page_size * (page - 1))
)] )]
count = conn.execute( count = conn.execute(
""" "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ?",
SELECT COUNT(*) (claim_id,)
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ?
""", (claim_id,)
) )
else: else:
results = [clean(dict(row)) for row in conn.execute( results = [clean(dict(row)) for row in conn.execute(
""" SELECT comment, comment_id, channel_name, channel_id, SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? ",
channel_url, timestamp, signature, signing_ts, parent_id
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ? AND parent_id = ?
LIMIT ? OFFSET ? """,
(claim_id, parent_id, page_size, page_size * (page - 1)) (claim_id, parent_id, page_size, page_size * (page - 1))
)] )]
count = conn.execute( count = conn.execute(
""" "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id = ?",
SELECT COUNT(*) (claim_id, parent_id)
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ? AND parent_id = ?
""", (claim_id, parent_id)
) )
count = tuple(count.fetchone())[0] count = tuple(count.fetchone())[0]
return { return {
@ -80,10 +74,42 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
'page': page, 'page': page,
'page_size': page_size, 'page_size': page_size,
'total_pages': math.ceil(count / page_size), 'total_pages': math.ceil(count / page_size),
'total_items': count 'total_items': count,
'has_hidden_comments': claim_has_hidden_comments(conn, claim_id)
} }
def get_claim_hidden_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50):
with conn:
results = conn.execute(
SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden = ? LIMIT ? OFFSET ?",
(claim_id, hidden, page_size, page_size * (page - 1))
)
count = conn.execute(
"SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = ?", (claim_id, hidden)
)
results = [clean(dict(row)) for row in results.fetchall()]
count = tuple(count.fetchone())[0]
return {
'items': results,
'page': page,
'page_size': page_size,
'total_pages': math.ceil(count/page_size),
'total_items': count,
'has_hidden_comments': claim_has_hidden_comments(conn, claim_id)
}
def claim_has_hidden_comments(conn, claim_id):
with conn:
result = conn.execute(
"SELECT COUNT(DISTINCT is_hidden) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = TRUE",
(claim_id,)
)
return bool(tuple(result.fetchone())[0])
def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent_id: str = None, def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent_id: str = None,
channel_id: str = None, signature: str = None, signing_ts: str = None) -> str: channel_id: str = None, signature: str = None, signing_ts: str = None) -> str:
timestamp = int(time.time()) timestamp = int(time.time())
@ -103,13 +129,7 @@ def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent
def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict: def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict:
with conn: with conn:
curry = conn.execute( curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS_CLAIMID + "WHERE comment_id = ?", (comment_id,))
"""
SELECT comment, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id
FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?
""",
(comment_id,)
)
thing = curry.fetchone() thing = curry.fetchone()
return clean(dict(thing)) if thing else None return clean(dict(thing)) if thing else None
@ -138,26 +158,17 @@ def get_comment_ids(conn: sqlite3.Connection, claim_id: str, parent_id: str = No
return [tuple(row)[0] for row in curs.fetchall()] return [tuple(row)[0] for row in curs.fetchall()]
def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]: def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.Union[list, None]:
""" Returns a list containing the comment data associated with each ID within the list""" """ Returns a list containing the comment data associated with each ID within the list"""
# format the input, under the assumption that the # format the input, under the assumption that the
placeholders = ', '.join('?' for _ in comment_ids) placeholders = ', '.join('?' for _ in comment_ids)
with conn: with conn:
return [clean(dict(row)) for row in conn.execute( return [clean(dict(row)) for row in conn.execute(
f'SELECT * FROM COMMENTS_ON_CLAIMS WHERE comment_id IN ({placeholders})', SELECT_COMMENTS_ON_CLAIMS_CLAIMID + f'WHERE comment_id IN ({placeholders})',
tuple(comment_ids) tuple(comment_ids)
)] )]
def delete_anonymous_comment_by_id(conn: sqlite3.Connection, comment_id: str):
with conn:
curs = conn.execute(
"DELETE FROM COMMENT WHERE ChannelId IS NULL AND CommentId = ?",
(comment_id,)
)
return curs.rowcount
def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str): def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str):
with conn: with conn:
curs = conn.execute("DELETE FROM COMMENT WHERE CommentId = ?", (comment_id,)) curs = conn.execute("DELETE FROM COMMENT WHERE CommentId = ?", (comment_id,))
@ -166,19 +177,36 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str):
def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str): def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str):
with conn: with conn:
conn.execute( curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', (channel_id, channel_name))
'INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', return bool(curs.rowcount)
(channel_id, channel_name)
)
def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str): def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str):
with conn: with conn:
channel = conn.execute(""" channel = conn.execute(
SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ? "SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?", (comment_id,)
""", (comment_id,)
).fetchone() ).fetchone()
return dict(channel) if channel else dict() return dict(channel) if channel else {}
def get_claim_ids_from_comment_ids(conn: sqlite3.Connection, comment_ids: list):
with conn:
cids = conn.execute(
f""" SELECT CommentId as comment_id, LbryClaimId AS claim_id FROM COMMENT
WHERE CommentId IN ({', '.join('?' for _ in comment_ids)}) """,
tuple(comment_ids)
)
return {row['comment_id']: row['claim_id'] for row in cids.fetchall()}
def hide_comments_by_id(conn: sqlite3.Connection, comment_ids: list):
with conn:
curs = conn.cursor()
curs.executemany(
"UPDATE COMMENT SET IsHidden = TRUE WHERE CommentId = ?",
[[c] for c in comment_ids]
)
return bool(curs.rowcount)
class DatabaseWriter(object): class DatabaseWriter(object):

12
src/database/schema.py
View file

@ -6,12 +6,13 @@ CREATE_COMMENT_TABLE = """
CREATE TABLE IF NOT EXISTS COMMENT ( CREATE TABLE IF NOT EXISTS COMMENT (
CommentId TEXT NOT NULL, CommentId TEXT NOT NULL,
LbryClaimId TEXT NOT NULL, LbryClaimId TEXT NOT NULL,
ChannelId TEXT DEFAULT NULL, ChannelId TEXT DEFAULT NULL,
Body TEXT NOT NULL, Body TEXT NOT NULL,
ParentId TEXT DEFAULT NULL, ParentId TEXT DEFAULT NULL,
Signature TEXT DEFAULT NULL, Signature TEXT DEFAULT NULL,
Timestamp INTEGER NOT NULL, Timestamp INTEGER NOT NULL,
SigningTs TEXT DEFAULT NULL, SigningTs TEXT DEFAULT NULL,
IsHidden BOOLEAN NOT NULL DEFAULT (FALSE),
CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE, CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT, CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId) CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
@ -46,7 +47,8 @@ CREATE_COMMENTS_ON_CLAIMS_VIEW = """
('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url, ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
C.Signature AS signature, C.Signature AS signature,
C.SigningTs AS signing_ts, C.SigningTs AS signing_ts,
C.ParentId AS parent_id C.ParentId AS parent_id,
C.IsHidden as is_hidden
FROM COMMENT AS C FROM COMMENT AS C
LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
ORDER BY C.Timestamp DESC; ORDER BY C.Timestamp DESC;

48
src/database/writes.py
View file

@ -3,12 +3,15 @@ import sqlite3
from asyncio import coroutine from asyncio import coroutine
from database.queries import delete_comment_by_id from src.database.queries import hide_comments_by_id
from src.database.queries import delete_comment_by_id
from src.database.queries import get_comment_or_none
from src.database.queries import insert_comment
from src.database.queries import insert_channel
from src.database.queries import get_claim_ids_from_comment_ids
from src.server.misc import is_authentic_delete_signal from src.server.misc import is_authentic_delete_signal
from src.server.misc import request_lbrynet
from database.queries import get_comment_or_none from src.server.misc import validate_signature_from_claim
from database.queries import insert_comment
from database.queries import insert_channel
from src.server.misc import channel_matches_pattern_or_error from src.server.misc import channel_matches_pattern_or_error
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -47,5 +50,36 @@ async def delete_comment_if_authorized(app, comment_id, **kwargs):
return {'deleted': await job.wait()} return {'deleted': await job.wait()}
async def write_comment(app, comment): async def write_comment(app, params):
return await coroutine(create_comment_or_error)(app['writer'], **comment) return await coroutine(create_comment_or_error)(app['writer'], **params)
async def hide_comments(app, comment_ids):
return await coroutine(hide_comments_by_id)(app['writer'], comment_ids)
async def claim_search(app, **kwargs):
return (await request_lbrynet(app, 'claim_search', **kwargs))['items'][0]
async def hide_comments_where_authorized(app, pieces: list):
comment_cids = get_claim_ids_from_comment_ids(
conn=app['reader'],
comment_ids=[p['comment_id'] for p in pieces]
)
# TODO: Amortize this process
claims = {}
comments_to_hide = []
for p in pieces:
claim_id = comment_cids[p['comment_id']]
if claim_id not in claims:
claims[claim_id] = await claim_search(app, claim_id=claim_id, no_totals=True)
channel = claims[claim_id].get('signing_channel')
if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']):
comments_to_hide.append(p['comment_id'])
if comments_to_hide:
job = await app['comment_scheduler'].spawn(hide_comments(app, comments_to_hide))
await job.wait()
return {'hidden': comments_to_hide}

4
src/server/app.py
View file

@ -9,8 +9,8 @@ import aiojobs.aiohttp
import asyncio import asyncio
from aiohttp import web from aiohttp import web
from database.queries import setup_database, backup_database from src.database.queries import setup_database, backup_database
from database.queries import obtain_connection, DatabaseWriter from src.database.queries import obtain_connection, DatabaseWriter
from src.server.handles import api_endpoint, get_api_endpoint from src.server.handles import api_endpoint, get_api_endpoint
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

38
src/server/handles.py
View file

@ -7,13 +7,17 @@ from aiohttp import web
from aiojobs.aiohttp import atomic from aiojobs.aiohttp import atomic
from src.server.misc import clean_input_params from src.server.misc import clean_input_params
from database.queries import get_claim_comments from src.database.queries import get_claim_comments
from database.queries import get_comments_by_id, get_comment_ids from src.database.queries import get_comments_by_id, get_comment_ids
from database.queries import get_channel_id_from_comment_id from src.database.queries import get_channel_id_from_comment_id
from src.database.queries import get_claim_hidden_comments
from src.server.misc import is_valid_base_comment from src.server.misc import is_valid_base_comment
from src.server.misc import is_valid_credential_input from src.server.misc import is_valid_credential_input
from src.server.misc import make_error from src.server.misc import make_error
from database.writes import delete_comment_if_authorized, write_comment from src.database.writes import delete_comment_if_authorized
from src.database.writes import write_comment
from src.database.writes import hide_comments_where_authorized
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -24,23 +28,23 @@ def ping(*args):
def handle_get_channel_from_comment_id(app, kwargs: dict): def handle_get_channel_from_comment_id(app, kwargs: dict):
with app['reader'] as conn: return get_channel_id_from_comment_id(app['reader'], **kwargs)
return get_channel_id_from_comment_id(conn, **kwargs)
def handle_get_comment_ids(app, kwargs): def handle_get_comment_ids(app, kwargs):
with app['reader'] as conn: return get_comment_ids(app['reader'], **kwargs)
return get_comment_ids(conn, **kwargs)
def handle_get_claim_comments(app, kwargs): def handle_get_claim_comments(app, kwargs):
with app['reader'] as conn: return get_claim_comments(app['reader'], **kwargs)
return get_claim_comments(conn, **kwargs)
def handle_get_comments_by_id(app, kwargs): def handle_get_comments_by_id(app, kwargs):
with app['reader'] as conn: return get_comments_by_id(app['reader'], **kwargs)
return get_comments_by_id(conn, **kwargs)
def handle_get_claim_hidden_comments(app, kwargs):
return get_claim_hidden_comments(app['reader'], **kwargs)
async def handle_create_comment(app, params): async def handle_create_comment(app, params):
@ -55,15 +59,21 @@ async def handle_delete_comment(app, params):
return await delete_comment_if_authorized(app, **params) return await delete_comment_if_authorized(app, **params)
async def handle_hide_comments(app, params):
return await hide_comments_where_authorized(app, **params)
METHODS = { METHODS = {
'ping': ping, 'ping': ping,
'get_claim_comments': handle_get_claim_comments, 'get_claim_comments': handle_get_claim_comments,
'get_claim_hidden_comments': handle_get_claim_hidden_comments,
'get_comment_ids': handle_get_comment_ids, 'get_comment_ids': handle_get_comment_ids,
'get_comments_by_id': handle_get_comments_by_id, 'get_comments_by_id': handle_get_comments_by_id,
'get_channel_from_comment_id': handle_get_channel_from_comment_id, 'get_channel_from_comment_id': handle_get_channel_from_comment_id,
'create_comment': handle_create_comment, 'create_comment': handle_create_comment,
'delete_comment': handle_delete_comment, 'delete_comment': handle_delete_comment,
# 'abandon_comment': handle_delete_comment, 'abandon_comment': handle_delete_comment,
'hide_comments': handle_hide_comments
} }
@ -98,7 +108,7 @@ async def process_json(app, body: dict) -> dict:
@atomic @atomic
async def api_endpoint(request: web.Request): async def api_endpoint(request: web.Request):
try: try:
web.access_logger.info(f'Forwarded headers: {request.forwarded}') web.access_logger.info(f'Forwarded headers: {request.remote}')
body = await request.json() body = await request.json()
if type(body) is list or type(body) is dict: if type(body) is list or type(body) is dict:
if type(body) is list: if type(body) is list:

31
src/server/misc.py
View file

@ -41,20 +41,19 @@ def make_error(error, exc=None) -> dict:
return body return body
async def resolve_channel_claim(app, channel_id, channel_name): async def request_lbrynet(app, method, **params):
lbry_url = f'lbry://{channel_name}#{channel_id}' body = {'method': method, 'params': {**params}}
resolve_body = {'method': 'resolve', 'params': {'urls': [lbry_url]}}
try: try:
async with aiohttp.request('POST', app['config']['LBRYNET'], json=resolve_body) as req: async with aiohttp.request('POST', app['config']['LBRYNET'], json=body) as req:
try: try:
resp = await req.json() resp = await req.json()
except JSONDecodeError as jde: except JSONDecodeError as jde:
logger.exception(jde.msg) logger.exception(jde.msg)
raise Exception('JSON Decode Error in Claim Resolution') raise Exception('JSON Decode Error In lbrynet request')
finally: finally:
if 'result' in resp: if 'result' in resp:
return resp['result'].get(lbry_url) return resp['result']
raise ValueError('claim resolution yields error', {'error': resp['error']}) raise ValueError('LBRYNET Request Error', {'error': resp['error']})
except (ConnectionRefusedError, ClientConnectorError): except (ConnectionRefusedError, ClientConnectorError):
logger.critical("Connection to the LBRYnet daemon failed, make sure it's running.") logger.critical("Connection to the LBRYnet daemon failed, make sure it's running.")
raise Exception("Server cannot verify delete signature") raise Exception("Server cannot verify delete signature")
@ -111,7 +110,8 @@ def is_valid_credential_input(channel_id=None, channel_name=None, signature=None
async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id, signature, signing_ts): async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id, signature, signing_ts):
claim = await resolve_channel_claim(app, channel_id, channel_name) lbry_url = f'lbry://{channel_name}#{channel_id}'
claim = await request_lbrynet(app, 'resolve', urls=[lbry_url])
if claim: if claim:
public_key = claim['value']['public_key'] public_key = claim['value']['public_key']
claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1] claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1]
@ -124,6 +124,21 @@ async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id,
return False return False
def validate_signature_from_claim(claim, signature, signing_ts, data: str):
try:
if claim:
public_key = claim['value']['public_key']
claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1]
injest = b''.join((signing_ts.encode(), claim_hash, data.encode()))
return is_signature_valid(
encoded_signature=get_encoded_signature(signature),
signature_digest=hashlib.sha256(injest).digest(),
public_key_bytes=binascii.unhexlify(public_key.encode())
)
except:
return False
def clean_input_params(kwargs: dict): def clean_input_params(kwargs: dict):
for k, v in kwargs.items(): for k, v in kwargs.items():
if type(v) is str: if type(v) is str:

80
tests/database_test.py
View file

@ -5,10 +5,13 @@ from faker.providers import internet
from faker.providers import lorem from faker.providers import lorem
from faker.providers import misc from faker.providers import misc
from database.queries import get_comments_by_id from src.database.queries import get_comments_by_id
from database.queries import get_comment_ids from src.database.queries import get_comment_ids
from database.queries import get_claim_comments from src.database.queries import get_claim_comments
from database.writes import create_comment_or_error from src.database.queries import get_claim_hidden_comments
from src.database.writes import create_comment_or_error
from src.database.queries import hide_comments_by_id
from src.database.queries import delete_comment_by_id
from tests.testcase import DatabaseTestCase from tests.testcase import DatabaseTestCase
fake = faker.Faker() fake = faker.Faker()
@ -17,7 +20,7 @@ fake.add_provider(lorem)
fake.add_provider(misc) fake.add_provider(misc)
class TestCommentCreation(DatabaseTestCase): class TestDatabaseOperations(DatabaseTestCase):
def setUp(self) -> None: def setUp(self) -> None:
super().setUp() super().setUp()
self.claimId = '529357c3422c6046d3fec76be2358004ba22e340' self.claimId = '529357c3422c6046d3fec76be2358004ba22e340'
@ -192,6 +195,32 @@ class TestCommentCreation(DatabaseTestCase):
self.assertLessEqual(len(replies), 50) self.assertLessEqual(len(replies), 50)
self.assertEqual(len(replies), len(comments_ids)) self.assertEqual(len(replies), len(comments_ids))
def test07HideComments(self):
comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
self.assertFalse(comment['is_hidden'])
success = hide_comments_by_id(self.conn, [comm['comment_id']])
self.assertTrue(success)
comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
self.assertTrue(comment['is_hidden'])
success = hide_comments_by_id(self.conn, [comm['comment_id']])
self.assertTrue(success)
comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
self.assertTrue(comment['is_hidden'])
def test08DeleteComments(self):
comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
comments = get_claim_comments(self.conn, self.claimId)
match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items']))
self.assertTrue(match)
deleted = delete_comment_by_id(self.conn, comm['comment_id'])
self.assertTrue(deleted)
comments = get_claim_comments(self.conn, self.claimId)
match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items']))
self.assertFalse(match)
deleted = delete_comment_by_id(self.conn, comm['comment_id'])
self.assertFalse(deleted)
class ListDatabaseTest(DatabaseTestCase): class ListDatabaseTest(DatabaseTestCase):
def setUp(self) -> None: def setUp(self) -> None:
@ -204,6 +233,8 @@ class ListDatabaseTest(DatabaseTestCase):
comments = get_claim_comments(self.conn, claim_id) comments = get_claim_comments(self.conn, claim_id)
self.assertIsNotNone(comments) self.assertIsNotNone(comments)
self.assertGreater(comments['page_size'], 0) self.assertGreater(comments['page_size'], 0)
self.assertIn('has_hidden_comments', comments)
self.assertFalse(comments['has_hidden_comments'])
top_comments = get_claim_comments(self.conn, claim_id, top_level=True, page=1, page_size=50) top_comments = get_claim_comments(self.conn, claim_id, top_level=True, page=1, page_size=50)
self.assertIsNotNone(top_comments) self.assertIsNotNone(top_comments)
self.assertEqual(top_comments['page_size'], 50) self.assertEqual(top_comments['page_size'], 50)
@ -218,6 +249,45 @@ class ListDatabaseTest(DatabaseTestCase):
self.assertIsNotNone(matching_comments) self.assertIsNotNone(matching_comments)
self.assertEqual(len(matching_comments), len(comment_ids)) self.assertEqual(len(matching_comments), len(comment_ids))
def testHiddenCommentLists(self):
claim_id = 'a'*40
comm1 = create_comment_or_error(self.conn, 'Comment #1', claim_id, '1'*40, '@Doge123', 'a'*128, '123')
comm2 = create_comment_or_error(self.conn, 'Comment #2', claim_id, '1'*40, '@Doge123', 'b'*128, '123')
comm3 = create_comment_or_error(self.conn, 'Comment #3', claim_id, '1'*40, '@Doge123', 'c'*128, '123')
comments = [comm1, comm2, comm3]
comment_list = get_claim_comments(self.conn, claim_id)
self.assertIn('items', comment_list)
self.assertIn('has_hidden_comments', comment_list)
self.assertEqual(len(comments), comment_list['total_items'])
self.assertIn('has_hidden_comments', comment_list)
self.assertFalse(comment_list['has_hidden_comments'])
hide_comments_by_id(self.conn, [comm2['comment_id']])
default_comments = get_claim_hidden_comments(self.conn, claim_id)
self.assertIn('has_hidden_comments', default_comments)
hidden_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=True)
self.assertIn('has_hidden_comments', hidden_comments)
self.assertEqual(default_comments, hidden_comments)
hidden_comment = hidden_comments['items'][0]
self.assertEqual(hidden_comment['comment_id'], comm2['comment_id'])
visible_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=False)
self.assertIn('has_hidden_comments', visible_comments)
self.assertNotIn(hidden_comment, visible_comments['items'])
hidden_ids = [c['comment_id'] for c in hidden_comments['items']]
visible_ids = [c['comment_id'] for c in visible_comments['items']]
composite_ids = hidden_ids + visible_ids
composite_ids.sort()
comment_list = get_claim_comments(self.conn, claim_id)
all_ids = [c['comment_id'] for c in comment_list['items']]
all_ids.sort()
self.assertEqual(composite_ids, all_ids)
def generate_top_comments(ncid=15, ncomm=100, minchar=50, maxchar=500): def generate_top_comments(ncid=15, ncomm=100, minchar=50, maxchar=500):
claim_ids = [fake.sha1() for _ in range(ncid)] claim_ids = [fake.sha1() for _ in range(ncid)]

2
tests/testcase.py
View file

@ -6,7 +6,7 @@ from unittest.case import _Outcome
import asyncio import asyncio
from database.queries import obtain_connection, setup_database from src.database.queries import obtain_connection, setup_database
class AsyncioTestCase(unittest.TestCase): class AsyncioTestCase(unittest.TestCase):