From 87150d94703ca35d552e9f519ea948f8784d639e Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sat, 3 Aug 2019 23:32:37 -0400
Subject: [PATCH 01/21] Adds `is_hidden` to schema
---
src/database/comments_ddl.sql | 45 ++++++++++++++++++-----------------
src/database/queries.py | 18 +++++++++-----
src/database/schema.py | 12 ++++++----
3 files changed, 42 insertions(+), 33 deletions(-)
diff --git a/src/database/comments_ddl.sql b/src/database/comments_ddl.sql
index 7556fbd..fc85acf 100644
--- a/src/database/comments_ddl.sql
+++ b/src/database/comments_ddl.sql
@@ -9,12 +9,13 @@ CREATE TABLE IF NOT EXISTS COMMENT
(
CommentId TEXT NOT NULL,
LbryClaimId TEXT NOT NULL,
- ChannelId TEXT DEFAULT NULL,
+ ChannelId TEXT DEFAULT (NULL),
Body TEXT NOT NULL,
- ParentId TEXT DEFAULT NULL,
- Signature TEXT DEFAULT NULL,
+ ParentId TEXT DEFAULT (NULL),
+ Signature TEXT DEFAULT (NULL),
Timestamp INTEGER NOT NULL,
- SigningTs TEXT DEFAULT NULL,
+ SigningTs TEXT DEFAULT (NULL),
+ IsHidden BOOLEAN NOT NULL DEFAULT (FALSE),
CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
@@ -23,6 +24,7 @@ CREATE TABLE IF NOT EXISTS COMMENT
ON UPDATE CASCADE ON DELETE NO ACTION -- setting null implies comment is top level
);
+-- ALTER TABLE COMMENT ADD COLUMN IsHidden BOOLEAN DEFAULT (FALSE);
-- ALTER TABLE COMMENT ADD COLUMN SigningTs TEXT DEFAULT NULL;
-- DROP TABLE IF EXISTS CHANNEL;
@@ -37,27 +39,26 @@ CREATE TABLE IF NOT EXISTS CHANNEL
-- indexes
-- DROP INDEX IF EXISTS COMMENT_CLAIM_INDEX;
-CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
+-- CREATE INDEX IF NOT EXISTS CLAIM_COMMENT_INDEX ON COMMENT (LbryClaimId, CommentId);
-CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
+-- CREATE INDEX IF NOT EXISTS CHANNEL_COMMENT_INDEX ON COMMENT (ChannelId, CommentId);
-- VIEWS
-CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS (comment_id, claim_id, timestamp, channel_name, channel_id, channel_url,
- signature, signing_ts, parent_id, comment) AS
-SELECT C.CommentId,
- C.LbryClaimId,
- C.Timestamp,
- CHAN.Name,
- CHAN.ClaimId,
- 'lbry://' || CHAN.Name || '#' || CHAN.ClaimId,
- C.Signature,
- C.SigningTs,
- C.ParentId,
- C.Body
-FROM COMMENT AS C
- LEFT OUTER JOIN CHANNEL CHAN on C.ChannelId = CHAN.ClaimId
-ORDER BY C.Timestamp DESC;
-
+CREATE VIEW IF NOT EXISTS COMMENTS_ON_CLAIMS AS SELECT
+ C.CommentId AS comment_id,
+ C.Body AS comment,
+ C.LbryClaimId AS claim_id,
+ C.Timestamp AS timestamp,
+ CHAN.Name AS channel_name,
+ CHAN.ClaimId AS channel_id,
+ ('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
+ C.Signature AS signature,
+ C.SigningTs AS signing_ts,
+ C.ParentId AS parent_id,
+ C.IsHidden AS is_hidden
+ FROM COMMENT AS C
+ LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
+ ORDER BY C.Timestamp DESC;
DROP VIEW IF EXISTS COMMENT_REPLIES;
diff --git a/src/database/queries.py b/src/database/queries.py
index 417cf15..d26be9e 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -13,6 +13,8 @@ logger = logging.getLogger(__name__)
def clean(thing: dict) -> dict:
+ if 'is_hidden' in thing:
+ thing.update({'is_hidden': bool(thing['is_hidden'])})
return {k: v for k, v in thing.items() if v}
@@ -29,7 +31,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
if top_level:
results = [clean(dict(row)) for row in conn.execute(
""" SELECT comment, comment_id, channel_name, channel_id,
- channel_url, timestamp, signature, signing_ts, parent_id
+ channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ? AND parent_id IS NULL
LIMIT ? OFFSET ? """,
@@ -45,7 +47,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
elif parent_id is None:
results = [clean(dict(row)) for row in conn.execute(
""" SELECT comment, comment_id, channel_name, channel_id,
- channel_url, timestamp, signature, signing_ts, parent_id
+ channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ?
LIMIT ? OFFSET ? """,
@@ -61,7 +63,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
else:
results = [clean(dict(row)) for row in conn.execute(
""" SELECT comment, comment_id, channel_name, channel_id,
- channel_url, timestamp, signature, signing_ts, parent_id
+ channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
FROM COMMENTS_ON_CLAIMS
WHERE claim_id = ? AND parent_id = ?
LIMIT ? OFFSET ? """,
@@ -105,7 +107,7 @@ def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict:
with conn:
curry = conn.execute(
"""
- SELECT comment, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id
+ SELECT comment, claim_id, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?
""",
(comment_id,)
@@ -144,7 +146,11 @@ def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]:
placeholders = ', '.join('?' for _ in comment_ids)
with conn:
return [clean(dict(row)) for row in conn.execute(
- f'SELECT * FROM COMMENTS_ON_CLAIMS WHERE comment_id IN ({placeholders})',
+ """
+ SELECT comment, claim_id, comment_id, channel_name, channel_id,
+ channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
+ FROM COMMENTS_ON_CLAIMS
+ """ + f' WHERE comment_id IN ({placeholders})',
tuple(comment_ids)
)]
@@ -178,7 +184,7 @@ def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str):
SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?
""", (comment_id,)
).fetchone()
- return dict(channel) if channel else dict()
+ return dict(channel) if channel else {}
class DatabaseWriter(object):
diff --git a/src/database/schema.py b/src/database/schema.py
index bf46f72..f474765 100644
--- a/src/database/schema.py
+++ b/src/database/schema.py
@@ -6,12 +6,13 @@ CREATE_COMMENT_TABLE = """
CREATE TABLE IF NOT EXISTS COMMENT (
CommentId TEXT NOT NULL,
LbryClaimId TEXT NOT NULL,
- ChannelId TEXT DEFAULT NULL,
+ ChannelId TEXT DEFAULT NULL,
Body TEXT NOT NULL,
- ParentId TEXT DEFAULT NULL,
- Signature TEXT DEFAULT NULL,
+ ParentId TEXT DEFAULT NULL,
+ Signature TEXT DEFAULT NULL,
Timestamp INTEGER NOT NULL,
- SigningTs TEXT DEFAULT NULL,
+ SigningTs TEXT DEFAULT NULL,
+ IsHidden BOOLEAN NOT NULL DEFAULT (FALSE),
CONSTRAINT COMMENT_PRIMARY_KEY PRIMARY KEY (CommentId) ON CONFLICT IGNORE,
CONSTRAINT COMMENT_SIGNATURE_SK UNIQUE (Signature) ON CONFLICT ABORT,
CONSTRAINT COMMENT_CHANNEL_FK FOREIGN KEY (ChannelId) REFERENCES CHANNEL (ClaimId)
@@ -46,7 +47,8 @@ CREATE_COMMENTS_ON_CLAIMS_VIEW = """
('lbry://' || CHAN.Name || '#' || CHAN.ClaimId) AS channel_url,
C.Signature AS signature,
C.SigningTs AS signing_ts,
- C.ParentId AS parent_id
+ C.ParentId AS parent_id,
+ C.IsHidden as is_hidden
FROM COMMENT AS C
LEFT OUTER JOIN CHANNEL CHAN ON C.ChannelId = CHAN.ClaimId
ORDER BY C.Timestamp DESC;
From 3c9e9d13c91ed323c9c22be3c4cd6a392180fcc4 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sat, 3 Aug 2019 23:34:00 -0400
Subject: [PATCH 02/21] Adds hide comment query & function
---
src/database/queries.py | 9 +++++++++
src/database/writes.py | 33 ++++++++++++++++++++++++++++++---
2 files changed, 39 insertions(+), 3 deletions(-)
diff --git a/src/database/queries.py b/src/database/queries.py
index d26be9e..568bc35 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -170,6 +170,15 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str):
return bool(curs.rowcount)
+def hide_comment_by_id(conn: sqlite3.Connection, comment_id: str):
+ with conn:
+ curs = conn.execute("""
+ UPDATE OR IGNORE COMMENT SET IsHidden = TRUE
+ WHERE CommentId = ?
+ """, (comment_id,))
+ return bool(curs.rowcount)
+
+
def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str):
with conn:
conn.execute(
diff --git a/src/database/writes.py b/src/database/writes.py
index 0c4ba03..7fe99c2 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -4,11 +4,13 @@ import sqlite3
from asyncio import coroutine
from database.queries import delete_comment_by_id
-from src.server.misc import is_authentic_delete_signal
+from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim
from database.queries import get_comment_or_none
from database.queries import insert_comment
from database.queries import insert_channel
+from database.queries import get_channel_id_from_comment_id
+from database.queries import hide_comment_by_id
from src.server.misc import channel_matches_pattern_or_error
logger = logging.getLogger(__name__)
@@ -47,5 +49,30 @@ async def delete_comment_if_authorized(app, comment_id, **kwargs):
return {'deleted': await job.wait()}
-async def write_comment(app, comment):
- return await coroutine(create_comment_or_error)(app['writer'], **comment)
+async def write_comment(app, params):
+ return await coroutine(create_comment_or_error)(app['writer'], **params)
+
+
+async def hide_comment(app, comment_id):
+ return await coroutine(hide_comment_by_id)(app['writer'], comment_id)
+
+
+# comment_ids: [
+# {
+# "comment_id": id,
+# "signing_ts": signing_ts,
+# "signature": signature
+# },
+# ...
+# ]
+async def hide_comment_if_authorized(app, comment_id, signing_ts, signature):
+ channel = get_channel_id_from_comment_id(app['reader'], comment_id)
+ claim = await request_lbrynet(app, 'claim_search', claim_id=channel['channel_id'])
+ claim = claim['items'][0]
+ if not validate_signature_from_claim(claim, signature, signing_ts, comment_id):
+ raise ValueError('Invalid Signature')
+
+ job = await app['comment_scheduler'].spawn(hide_comment(app, comment_id))
+ return {
+ 'hidden': await job.wait()
+ }
From 26c01930ef23228d70162302b41876636711ff2a Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sat, 3 Aug 2019 23:35:13 -0400
Subject: [PATCH 03/21] Adds handles for hiding comments & verifying message
validity
---
src/server/handles.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/server/handles.py b/src/server/handles.py
index f1d8f98..0ce8349 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -13,7 +13,10 @@ from database.queries import get_channel_id_from_comment_id
from src.server.misc import is_valid_base_comment
from src.server.misc import is_valid_credential_input
from src.server.misc import make_error
-from database.writes import delete_comment_if_authorized, write_comment
+from database.writes import delete_comment_if_authorized
+from database.writes import write_comment
+from database.writes import hide_comment_if_authorized
+
logger = logging.getLogger(__name__)
@@ -55,6 +58,10 @@ async def handle_delete_comment(app, params):
return await delete_comment_if_authorized(app, **params)
+async def handle_hide_comment(app, params):
+ return await hide_comment_if_authorized(app, **params)
+
+
METHODS = {
'ping': ping,
'get_claim_comments': handle_get_claim_comments,
@@ -63,6 +70,7 @@ METHODS = {
'get_channel_from_comment_id': handle_get_channel_from_comment_id,
'create_comment': handle_create_comment,
'delete_comment': handle_delete_comment,
+ 'hide_comment': handle_hide_comment,
# 'abandon_comment': handle_delete_comment,
}
From 8d38bbd7ec0a6a6a805060267c897cedd6498162 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sat, 3 Aug 2019 23:35:37 -0400
Subject: [PATCH 04/21] Adds templates for querying lbrynet & validating
signatures
---
src/server/misc.py | 31 +++++++++++++++++++++++--------
1 file changed, 23 insertions(+), 8 deletions(-)
diff --git a/src/server/misc.py b/src/server/misc.py
index d0efd66..07567b9 100644
--- a/src/server/misc.py
+++ b/src/server/misc.py
@@ -41,20 +41,19 @@ def make_error(error, exc=None) -> dict:
return body
-async def resolve_channel_claim(app, channel_id, channel_name):
- lbry_url = f'lbry://{channel_name}#{channel_id}'
- resolve_body = {'method': 'resolve', 'params': {'urls': [lbry_url]}}
+async def request_lbrynet(app, method, **params):
+ body = {'method': method, 'params': {**params}}
try:
- async with aiohttp.request('POST', app['config']['LBRYNET'], json=resolve_body) as req:
+ async with aiohttp.request('POST', app['config']['LBRYNET'], json=body) as req:
try:
resp = await req.json()
except JSONDecodeError as jde:
logger.exception(jde.msg)
- raise Exception('JSON Decode Error in Claim Resolution')
+ raise Exception('JSON Decode Error In lbrynet request')
finally:
if 'result' in resp:
- return resp['result'].get(lbry_url)
- raise ValueError('claim resolution yields error', {'error': resp['error']})
+ return resp['result']
+ raise ValueError('LBRYNET Request Error', {'error': resp['error']})
except (ConnectionRefusedError, ClientConnectorError):
logger.critical("Connection to the LBRYnet daemon failed, make sure it's running.")
raise Exception("Server cannot verify delete signature")
@@ -111,7 +110,8 @@ def is_valid_credential_input(channel_id=None, channel_name=None, signature=None
async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id, signature, signing_ts):
- claim = await resolve_channel_claim(app, channel_id, channel_name)
+ lbry_url = f'lbry://{channel_name}#{channel_id}'
+ claim = await request_lbrynet(app, 'resolve', urls=[lbry_url])
if claim:
public_key = claim['value']['public_key']
claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1]
@@ -124,6 +124,21 @@ async def is_authentic_delete_signal(app, comment_id, channel_name, channel_id,
return False
+def validate_signature_from_claim(claim, signature, signing_ts, data: str):
+ try:
+ if claim:
+ public_key = claim['value']['public_key']
+ claim_hash = binascii.unhexlify(claim['claim_id'].encode())[::-1]
+ injest = b''.join((signing_ts.encode(), claim_hash, data.encode()))
+ return is_signature_valid(
+ encoded_signature=get_encoded_signature(signature),
+ signature_digest=hashlib.sha256(injest).digest(),
+ public_key_bytes=binascii.unhexlify(public_key.encode())
+ )
+ except:
+ return False
+
+
def clean_input_params(kwargs: dict):
for k, v in kwargs.items():
if type(v) is str:
From 7f98417d9d561285ba186b5920c5ed93a55ddbff Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 17:45:15 -0400
Subject: [PATCH 05/21] Adds
---
src/database/queries.py | 85 +++++++++++------------------------------
1 file changed, 23 insertions(+), 62 deletions(-)
diff --git a/src/database/queries.py b/src/database/queries.py
index 568bc35..2dca72b 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -12,6 +12,13 @@ from src.database.schema import CREATE_TABLES_QUERY
logger = logging.getLogger(__name__)
+SELECT_COMMENTS_ON_CLAIMS = """
+ SELECT comment, comment_id, channel_name, channel_id, channel_url,
+ timestamp, signature, signing_ts, parent_id, is_hidden
+ FROM COMMENTS_ON_CLAIMS
+"""
+
+
def clean(thing: dict) -> dict:
if 'is_hidden' in thing:
thing.update({'is_hidden': bool(thing['is_hidden'])})
@@ -30,51 +37,30 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
with conn:
if top_level:
results = [clean(dict(row)) for row in conn.execute(
- """ SELECT comment, comment_id, channel_name, channel_id,
- channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
- FROM COMMENTS_ON_CLAIMS
- WHERE claim_id = ? AND parent_id IS NULL
- LIMIT ? OFFSET ? """,
+ SELECT_COMMENTS_ON_CLAIMS + " WHERE claim_id = ? AND parent_id IS NULL LIMIT ? OFFSET ?",
(claim_id, page_size, page_size * (page - 1))
)]
count = conn.execute(
- """
- SELECT COUNT(*)
- FROM COMMENTS_ON_CLAIMS
- WHERE claim_id = ? AND parent_id IS NULL
- """, (claim_id,)
+ "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id IS NULL",
+ (claim_id,)
)
elif parent_id is None:
results = [clean(dict(row)) for row in conn.execute(
- """ SELECT comment, comment_id, channel_name, channel_id,
- channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
- FROM COMMENTS_ON_CLAIMS
- WHERE claim_id = ?
- LIMIT ? OFFSET ? """,
+ SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? LIMIT ? OFFSET ? ",
(claim_id, page_size, page_size * (page - 1))
)]
count = conn.execute(
- """
- SELECT COUNT(*)
- FROM COMMENTS_ON_CLAIMS
- WHERE claim_id = ?
- """, (claim_id,)
+ "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ?",
+ (claim_id,)
)
else:
results = [clean(dict(row)) for row in conn.execute(
- """ SELECT comment, comment_id, channel_name, channel_id,
- channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
- FROM COMMENTS_ON_CLAIMS
- WHERE claim_id = ? AND parent_id = ?
- LIMIT ? OFFSET ? """,
+ SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND parent_id = ? LIMIT ? OFFSET ? ",
(claim_id, parent_id, page_size, page_size * (page - 1))
)]
count = conn.execute(
- """
- SELECT COUNT(*)
- FROM COMMENTS_ON_CLAIMS
- WHERE claim_id = ? AND parent_id = ?
- """, (claim_id, parent_id)
+ "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND parent_id = ?",
+ (claim_id, parent_id)
)
count = tuple(count.fetchone())[0]
return {
@@ -105,13 +91,7 @@ def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent
def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict:
with conn:
- curry = conn.execute(
- """
- SELECT comment, claim_id, comment_id, channel_name, channel_id, channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
- FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?
- """,
- (comment_id,)
- )
+ curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,))
thing = curry.fetchone()
return clean(dict(thing)) if thing else None
@@ -146,24 +126,11 @@ def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]:
placeholders = ', '.join('?' for _ in comment_ids)
with conn:
return [clean(dict(row)) for row in conn.execute(
- """
- SELECT comment, claim_id, comment_id, channel_name, channel_id,
- channel_url, timestamp, signature, signing_ts, parent_id, is_hidden
- FROM COMMENTS_ON_CLAIMS
- """ + f' WHERE comment_id IN ({placeholders})',
+ SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})',
tuple(comment_ids)
)]
-def delete_anonymous_comment_by_id(conn: sqlite3.Connection, comment_id: str):
- with conn:
- curs = conn.execute(
- "DELETE FROM COMMENT WHERE ChannelId IS NULL AND CommentId = ?",
- (comment_id,)
- )
- return curs.rowcount
-
-
def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str):
with conn:
curs = conn.execute("DELETE FROM COMMENT WHERE CommentId = ?", (comment_id,))
@@ -172,26 +139,20 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str):
def hide_comment_by_id(conn: sqlite3.Connection, comment_id: str):
with conn:
- curs = conn.execute("""
- UPDATE OR IGNORE COMMENT SET IsHidden = TRUE
- WHERE CommentId = ?
- """, (comment_id,))
+ curs = conn.execute("UPDATE OR IGNORE COMMENT SET IsHidden = TRUE WHERE CommentId = ?", (comment_id,))
return bool(curs.rowcount)
def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str):
with conn:
- conn.execute(
- 'INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)',
- (channel_id, channel_name)
- )
+ curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', (channel_id, channel_name))
+ return bool(curs.rowcount)
def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str):
with conn:
- channel = conn.execute("""
- SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?
- """, (comment_id,)
+ channel = conn.execute(
+ "SELECT channel_id, channel_name FROM COMMENTS_ON_CLAIMS WHERE comment_id = ?", (comment_id,)
).fetchone()
return dict(channel) if channel else {}
From 1b79aefb054411b526ac82367c83d9b4d773fe1c Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 17:46:33 -0400
Subject: [PATCH 06/21] Adds hidden comment queries
---
src/database/queries.py | 36 ++++++++++++++++++++++++++++++++++--
1 file changed, 34 insertions(+), 2 deletions(-)
diff --git a/src/database/queries.py b/src/database/queries.py
index 2dca72b..7dd780c 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -22,7 +22,7 @@ SELECT_COMMENTS_ON_CLAIMS = """
def clean(thing: dict) -> dict:
if 'is_hidden' in thing:
thing.update({'is_hidden': bool(thing['is_hidden'])})
- return {k: v for k, v in thing.items() if v}
+ return {k: v for k, v in thing.items() if v is not None}
def obtain_connection(filepath: str = None, row_factory: bool = True):
@@ -68,10 +68,42 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
'page': page,
'page_size': page_size,
'total_pages': math.ceil(count / page_size),
- 'total_items': count
+ 'total_items': count,
+ 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id)
}
+def get_hidden_claim_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50):
+ with conn:
+ results = conn.execute(
+ SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden = ? LIMIT ? OFFSET ?",
+ (claim_id, hidden, page_size, page_size * (page - 1))
+ )
+ count = conn.execute(
+ "SELECT COUNT(*) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = ?", (claim_id, hidden)
+ )
+ results = [clean(dict(row)) for row in results.fetchall()]
+ count = tuple(count.fetchone())[0]
+
+ return {
+ 'items': results,
+ 'page': page,
+ 'page_size': page_size,
+ 'total_pages': math.ceil(count/page_size),
+ 'total_items': count,
+ 'has_hidden_comments': claim_has_hidden_comments(conn, claim_id)
+ }
+
+
+def claim_has_hidden_comments(conn, claim_id):
+ with conn:
+ result = conn.execute(
+ "SELECT COUNT(DISTINCT is_hidden) FROM COMMENTS_ON_CLAIMS WHERE claim_id = ? AND is_hidden = TRUE",
+ (claim_id,)
+ )
+ return bool(tuple(result.fetchone())[0])
+
+
def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent_id: str = None,
channel_id: str = None, signature: str = None, signing_ts: str = None) -> str:
timestamp = int(time.time())
From c56562e7eaca20e8ff95747a6024e684a2a8ae40 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 17:47:11 -0400
Subject: [PATCH 07/21] Import formats
---
src/database/writes.py | 12 ++++++------
src/server/handles.py | 13 +++++++------
tests/database_test.py | 11 +++++++----
tests/testcase.py | 2 +-
4 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/src/database/writes.py b/src/database/writes.py
index 7fe99c2..df0d950 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -3,14 +3,14 @@ import sqlite3
from asyncio import coroutine
-from database.queries import delete_comment_by_id
+from src.database.queries import delete_comment_by_id
from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim
-from database.queries import get_comment_or_none
-from database.queries import insert_comment
-from database.queries import insert_channel
-from database.queries import get_channel_id_from_comment_id
-from database.queries import hide_comment_by_id
+from src.database.queries import get_comment_or_none
+from src.database.queries import insert_comment
+from src.database.queries import insert_channel
+from src.database.queries import get_channel_id_from_comment_id
+from src.database.queries import hide_comment_by_id
from src.server.misc import channel_matches_pattern_or_error
logger = logging.getLogger(__name__)
diff --git a/src/server/handles.py b/src/server/handles.py
index 0ce8349..af765ba 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -7,15 +7,16 @@ from aiohttp import web
from aiojobs.aiohttp import atomic
from src.server.misc import clean_input_params
-from database.queries import get_claim_comments
-from database.queries import get_comments_by_id, get_comment_ids
-from database.queries import get_channel_id_from_comment_id
+from src.database.queries import get_claim_comments
+from src.database.queries import get_comments_by_id, get_comment_ids
+from src.database.queries import get_channel_id_from_comment_id
+from src.database.queries import get_hidden_claim_comments
from src.server.misc import is_valid_base_comment
from src.server.misc import is_valid_credential_input
from src.server.misc import make_error
-from database.writes import delete_comment_if_authorized
-from database.writes import write_comment
-from database.writes import hide_comment_if_authorized
+from src.database.writes import delete_comment_if_authorized
+from src.database.writes import write_comment
+from src.database.writes import hide_comment_if_authorized
logger = logging.getLogger(__name__)
diff --git a/tests/database_test.py b/tests/database_test.py
index 3de9312..4d9068c 100644
--- a/tests/database_test.py
+++ b/tests/database_test.py
@@ -5,10 +5,13 @@ from faker.providers import internet
from faker.providers import lorem
from faker.providers import misc
-from database.queries import get_comments_by_id
-from database.queries import get_comment_ids
-from database.queries import get_claim_comments
-from database.writes import create_comment_or_error
+from src.database.queries import get_comments_by_id
+from src.database.queries import get_comment_ids
+from src.database.queries import get_claim_comments
+from src.database.queries import get_hidden_claim_comments
+from src.database.writes import create_comment_or_error
+from src.database.queries import hide_comment_by_id
+from src.database.queries import delete_comment_by_id
from tests.testcase import DatabaseTestCase
fake = faker.Faker()
diff --git a/tests/testcase.py b/tests/testcase.py
index 2357347..ddd8b44 100644
--- a/tests/testcase.py
+++ b/tests/testcase.py
@@ -6,7 +6,7 @@ from unittest.case import _Outcome
import asyncio
-from database.queries import obtain_connection, setup_database
+from src.database.queries import obtain_connection, setup_database
class AsyncioTestCase(unittest.TestCase):
From 2b7e1df091c52d8ad00be6428d0bc2550487c437 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 17:47:53 -0400
Subject: [PATCH 08/21] Adds get_hidden_claim_comments handle & activates
abandon_comment func
---
src/server/handles.py | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/src/server/handles.py b/src/server/handles.py
index af765ba..662e13b 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -28,23 +28,23 @@ def ping(*args):
def handle_get_channel_from_comment_id(app, kwargs: dict):
- with app['reader'] as conn:
- return get_channel_id_from_comment_id(conn, **kwargs)
+ return get_channel_id_from_comment_id(app['reader'], **kwargs)
def handle_get_comment_ids(app, kwargs):
- with app['reader'] as conn:
- return get_comment_ids(conn, **kwargs)
+ return get_comment_ids(app['reader'], **kwargs)
def handle_get_claim_comments(app, kwargs):
- with app['reader'] as conn:
- return get_claim_comments(conn, **kwargs)
+ return get_claim_comments(app['reader'], **kwargs)
def handle_get_comments_by_id(app, kwargs):
- with app['reader'] as conn:
- return get_comments_by_id(conn, **kwargs)
+ return get_comments_by_id(app['reader'], **kwargs)
+
+
+def handle_get_hidden_claim_comments(app, kwargs):
+ return get_hidden_claim_comments(app['reader'], **kwargs)
async def handle_create_comment(app, params):
@@ -66,13 +66,14 @@ async def handle_hide_comment(app, params):
METHODS = {
'ping': ping,
'get_claim_comments': handle_get_claim_comments,
+ 'get_hidden_claim_comments': handle_get_hidden_claim_comments,
'get_comment_ids': handle_get_comment_ids,
'get_comments_by_id': handle_get_comments_by_id,
'get_channel_from_comment_id': handle_get_channel_from_comment_id,
'create_comment': handle_create_comment,
'delete_comment': handle_delete_comment,
+ 'abandon_comment': handle_delete_comment,
'hide_comment': handle_hide_comment,
- # 'abandon_comment': handle_delete_comment,
}
From 67d503acdbaac5d0b9353b5ab243ab2d5992e357 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 17:48:32 -0400
Subject: [PATCH 09/21] Adds unit tests for hidden comments + deleted comments
---
tests/database_test.py | 68 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 67 insertions(+), 1 deletion(-)
diff --git a/tests/database_test.py b/tests/database_test.py
index 4d9068c..02050a0 100644
--- a/tests/database_test.py
+++ b/tests/database_test.py
@@ -20,7 +20,7 @@ fake.add_provider(lorem)
fake.add_provider(misc)
-class TestCommentCreation(DatabaseTestCase):
+class TestDatabaseOperations(DatabaseTestCase):
def setUp(self) -> None:
super().setUp()
self.claimId = '529357c3422c6046d3fec76be2358004ba22e340'
@@ -195,6 +195,31 @@ class TestCommentCreation(DatabaseTestCase):
self.assertLessEqual(len(replies), 50)
self.assertEqual(len(replies), len(comments_ids))
+ def test07HideComments(self):
+ comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
+ comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
+ self.assertFalse(comment['is_hidden'])
+ success = hide_comment_by_id(self.conn, comm['comment_id'])
+ self.assertTrue(success)
+ comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
+ self.assertTrue(comment['is_hidden'])
+ success = hide_comment_by_id(self.conn, comm['comment_id'])
+ self.assertTrue(success)
+ comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
+ self.assertTrue(comment['is_hidden'])
+
+ def test08DeleteComments(self):
+ comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
+ comments = get_claim_comments(self.conn, self.claimId)
+ self.assertIn(comm, comments['items'])
+ deleted = delete_comment_by_id(self.conn, comm['comment_id'])
+ self.assertTrue(deleted)
+ comments = get_claim_comments(self.conn, self.claimId)
+ self.assertNotIn(comm, comments['items'])
+ deleted = delete_comment_by_id(self.conn, comm['comment_id'])
+ self.assertFalse(deleted)
+
+
class ListDatabaseTest(DatabaseTestCase):
def setUp(self) -> None:
@@ -207,6 +232,8 @@ class ListDatabaseTest(DatabaseTestCase):
comments = get_claim_comments(self.conn, claim_id)
self.assertIsNotNone(comments)
self.assertGreater(comments['page_size'], 0)
+ self.assertIn('has_hidden_comments', comments)
+ self.assertFalse(comments['has_hidden_comments'])
top_comments = get_claim_comments(self.conn, claim_id, top_level=True, page=1, page_size=50)
self.assertIsNotNone(top_comments)
self.assertEqual(top_comments['page_size'], 50)
@@ -221,6 +248,45 @@ class ListDatabaseTest(DatabaseTestCase):
self.assertIsNotNone(matching_comments)
self.assertEqual(len(matching_comments), len(comment_ids))
+ def testHiddenCommentLists(self):
+ claim_id = 'a'*40
+ comm1 = create_comment_or_error(self.conn, 'Comment #1', claim_id, '1'*40, '@Doge123', 'a'*128, '123')
+ comm2 = create_comment_or_error(self.conn, 'Comment #2', claim_id, '1'*40, '@Doge123', 'b'*128, '123')
+ comm3 = create_comment_or_error(self.conn, 'Comment #3', claim_id, '1'*40, '@Doge123', 'c'*128, '123')
+ comments = [comm1, comm2, comm3]
+
+ comment_list = get_claim_comments(self.conn, claim_id)
+ self.assertIn('items', comment_list)
+ self.assertIn('has_hidden_comments', comment_list)
+ self.assertEqual(len(comments), comment_list['total_items'])
+ self.assertIn('has_hidden_comments', comment_list)
+ self.assertFalse(comment_list['has_hidden_comments'])
+ hide_comment_by_id(self.conn, comm2['comment_id'])
+
+ default_comments = get_hidden_claim_comments(self.conn, claim_id)
+ self.assertIn('has_hidden_comments', default_comments)
+
+ hidden_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=True)
+ self.assertIn('has_hidden_comments', hidden_comments)
+ self.assertEqual(default_comments, hidden_comments)
+
+ hidden_comment = hidden_comments['items'][0]
+ self.assertEqual(hidden_comment['comment_id'], comm2['comment_id'])
+
+ visible_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=False)
+ self.assertIn('has_hidden_comments', visible_comments)
+ self.assertNotIn(hidden_comment, visible_comments['items'])
+
+ hidden_ids = [c['comment_id'] for c in hidden_comments['items']]
+ visible_ids = [c['comment_id'] for c in visible_comments['items']]
+ composite_ids = hidden_ids + visible_ids
+ composite_ids.sort()
+
+ comment_list = get_claim_comments(self.conn, claim_id)
+ all_ids = [c['comment_id'] for c in comment_list['items']]
+ all_ids.sort()
+ self.assertEqual(composite_ids, all_ids)
+
def generate_top_comments(ncid=15, ncomm=100, minchar=50, maxchar=500):
claim_ids = [fake.sha1() for _ in range(ncid)]
From 0f5ea0f88fcc49330863df1988423626f4c26f99 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 18:05:03 -0400
Subject: [PATCH 10/21] Fixes importerror
---
src/__init__.py | 2 --
src/database/__init__.py | 0
src/server/app.py | 4 ++--
3 files changed, 2 insertions(+), 4 deletions(-)
create mode 100644 src/database/__init__.py
diff --git a/src/__init__.py b/src/__init__.py
index b28b04f..8b13789 100644
--- a/src/__init__.py
+++ b/src/__init__.py
@@ -1,3 +1 @@
-
-
diff --git a/src/database/__init__.py b/src/database/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/server/app.py b/src/server/app.py
index 2512801..520120f 100644
--- a/src/server/app.py
+++ b/src/server/app.py
@@ -9,8 +9,8 @@ import aiojobs.aiohttp
import asyncio
from aiohttp import web
-from database.queries import setup_database, backup_database
-from database.queries import obtain_connection, DatabaseWriter
+from src.database.queries import setup_database, backup_database
+from src.database.queries import obtain_connection, DatabaseWriter
from src.server.handles import api_endpoint, get_api_endpoint
logger = logging.getLogger(__name__)
From f898173d9dffdb5551a28c7dae7f53bafb3a4100 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 18:06:13 -0400
Subject: [PATCH 11/21] Adds hide comment function to daemon
---
src/database/writes.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/database/writes.py b/src/database/writes.py
index df0d950..3e1bf67 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -4,13 +4,12 @@ import sqlite3
from asyncio import coroutine
from src.database.queries import delete_comment_by_id
-from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim
-
from src.database.queries import get_comment_or_none
from src.database.queries import insert_comment
from src.database.queries import insert_channel
from src.database.queries import get_channel_id_from_comment_id
from src.database.queries import hide_comment_by_id
+from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim
from src.server.misc import channel_matches_pattern_or_error
logger = logging.getLogger(__name__)
From d5dfd5a53b11fb2a27e7ce56ad848f7b479621aa Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 18:07:21 -0400
Subject: [PATCH 12/21] Revert "Adds hide comment function to daemon"
This reverts commit f898173d
---
src/database/writes.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/database/writes.py b/src/database/writes.py
index 3e1bf67..df0d950 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -4,12 +4,13 @@ import sqlite3
from asyncio import coroutine
from src.database.queries import delete_comment_by_id
+from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim
+
from src.database.queries import get_comment_or_none
from src.database.queries import insert_comment
from src.database.queries import insert_channel
from src.database.queries import get_channel_id_from_comment_id
from src.database.queries import hide_comment_by_id
-from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim
from src.server.misc import channel_matches_pattern_or_error
logger = logging.getLogger(__name__)
From 28944ad9b346c37f1cb70805ccaafb91d843c4cc Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 18:09:14 -0400
Subject: [PATCH 13/21] Fixes ImportError
---
src/database/writes.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/database/writes.py b/src/database/writes.py
index df0d950..a1cc57b 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -4,13 +4,14 @@ import sqlite3
from asyncio import coroutine
from src.database.queries import delete_comment_by_id
-from src.server.misc import is_authentic_delete_signal, request_lbrynet, validate_signature_from_claim
-
from src.database.queries import get_comment_or_none
from src.database.queries import insert_comment
from src.database.queries import insert_channel
from src.database.queries import get_channel_id_from_comment_id
from src.database.queries import hide_comment_by_id
+from src.server.misc import is_authentic_delete_signal
+from src.server.misc import request_lbrynet
+from src.server.misc import validate_signature_from_claim
from src.server.misc import channel_matches_pattern_or_error
logger = logging.getLogger(__name__)
From ffb3711ac920894bda54ccc01f2cbfee9738a9aa Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Sun, 4 Aug 2019 18:21:07 -0400
Subject: [PATCH 14/21] get_hidden_claim_comments -> get_claim_hidden_comments
---
src/database/queries.py | 2 +-
src/server/handles.py | 8 ++++----
tests/database_test.py | 8 ++++----
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/database/queries.py b/src/database/queries.py
index 7dd780c..4d0cf90 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -73,7 +73,7 @@ def get_claim_comments(conn: sqlite3.Connection, claim_id: str, parent_id: str =
}
-def get_hidden_claim_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50):
+def get_claim_hidden_comments(conn: sqlite3.Connection, claim_id: str, hidden=True, page=1, page_size=50):
with conn:
results = conn.execute(
SELECT_COMMENTS_ON_CLAIMS + "WHERE claim_id = ? AND is_hidden = ? LIMIT ? OFFSET ?",
diff --git a/src/server/handles.py b/src/server/handles.py
index 662e13b..5951960 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -10,7 +10,7 @@ from src.server.misc import clean_input_params
from src.database.queries import get_claim_comments
from src.database.queries import get_comments_by_id, get_comment_ids
from src.database.queries import get_channel_id_from_comment_id
-from src.database.queries import get_hidden_claim_comments
+from src.database.queries import get_claim_hidden_comments
from src.server.misc import is_valid_base_comment
from src.server.misc import is_valid_credential_input
from src.server.misc import make_error
@@ -43,8 +43,8 @@ def handle_get_comments_by_id(app, kwargs):
return get_comments_by_id(app['reader'], **kwargs)
-def handle_get_hidden_claim_comments(app, kwargs):
- return get_hidden_claim_comments(app['reader'], **kwargs)
+def handle_get_claim_hidden_comments(app, kwargs):
+ return get_claim_hidden_comments(app['reader'], **kwargs)
async def handle_create_comment(app, params):
@@ -66,7 +66,7 @@ async def handle_hide_comment(app, params):
METHODS = {
'ping': ping,
'get_claim_comments': handle_get_claim_comments,
- 'get_hidden_claim_comments': handle_get_hidden_claim_comments,
+ 'get_claim_hidden_comments': handle_get_claim_hidden_comments,
'get_comment_ids': handle_get_comment_ids,
'get_comments_by_id': handle_get_comments_by_id,
'get_channel_from_comment_id': handle_get_channel_from_comment_id,
diff --git a/tests/database_test.py b/tests/database_test.py
index 02050a0..ebf3477 100644
--- a/tests/database_test.py
+++ b/tests/database_test.py
@@ -8,7 +8,7 @@ from faker.providers import misc
from src.database.queries import get_comments_by_id
from src.database.queries import get_comment_ids
from src.database.queries import get_claim_comments
-from src.database.queries import get_hidden_claim_comments
+from src.database.queries import get_claim_hidden_comments
from src.database.writes import create_comment_or_error
from src.database.queries import hide_comment_by_id
from src.database.queries import delete_comment_by_id
@@ -263,17 +263,17 @@ class ListDatabaseTest(DatabaseTestCase):
self.assertFalse(comment_list['has_hidden_comments'])
hide_comment_by_id(self.conn, comm2['comment_id'])
- default_comments = get_hidden_claim_comments(self.conn, claim_id)
+ default_comments = get_claim_hidden_comments(self.conn, claim_id)
self.assertIn('has_hidden_comments', default_comments)
- hidden_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=True)
+ hidden_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=True)
self.assertIn('has_hidden_comments', hidden_comments)
self.assertEqual(default_comments, hidden_comments)
hidden_comment = hidden_comments['items'][0]
self.assertEqual(hidden_comment['comment_id'], comm2['comment_id'])
- visible_comments = get_hidden_claim_comments(self.conn, claim_id, hidden=False)
+ visible_comments = get_claim_hidden_comments(self.conn, claim_id, hidden=False)
self.assertIn('has_hidden_comments', visible_comments)
self.assertNotIn(hidden_comment, visible_comments['items'])
From 6d2447bd294180af7c2da97976dff4bdfbc9d674 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 9 Aug 2019 00:48:55 -0400
Subject: [PATCH 15/21] Adds queries for hiding comments + updates other
queries
---
src/database/queries.py | 38 +++++++++++++++++++++++++++++---------
1 file changed, 29 insertions(+), 9 deletions(-)
diff --git a/src/database/queries.py b/src/database/queries.py
index 4d0cf90..b4e2dd1 100644
--- a/src/database/queries.py
+++ b/src/database/queries.py
@@ -18,6 +18,12 @@ SELECT_COMMENTS_ON_CLAIMS = """
FROM COMMENTS_ON_CLAIMS
"""
+SELECT_COMMENTS_ON_CLAIMS_CLAIMID = """
+ SELECT comment, comment_id, claim_id, channel_name, channel_id, channel_url,
+ timestamp, signature, signing_ts, parent_id, is_hidden
+ FROM COMMENTS_ON_CLAIMS
+"""
+
def clean(thing: dict) -> dict:
if 'is_hidden' in thing:
@@ -123,7 +129,7 @@ def insert_comment(conn: sqlite3.Connection, claim_id: str, comment: str, parent
def get_comment_or_none(conn: sqlite3.Connection, comment_id: str) -> dict:
with conn:
- curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS + "WHERE comment_id = ?", (comment_id,))
+ curry = conn.execute(SELECT_COMMENTS_ON_CLAIMS_CLAIMID + "WHERE comment_id = ?", (comment_id,))
thing = curry.fetchone()
return clean(dict(thing)) if thing else None
@@ -152,13 +158,13 @@ def get_comment_ids(conn: sqlite3.Connection, claim_id: str, parent_id: str = No
return [tuple(row)[0] for row in curs.fetchall()]
-def get_comments_by_id(conn, comment_ids: list) -> typing.Union[list, None]:
+def get_comments_by_id(conn, comment_ids: typing.Union[list, tuple]) -> typing.Union[list, None]:
""" Returns a list containing the comment data associated with each ID within the list"""
# format the input, under the assumption that the
placeholders = ', '.join('?' for _ in comment_ids)
with conn:
return [clean(dict(row)) for row in conn.execute(
- SELECT_COMMENTS_ON_CLAIMS + f'WHERE comment_id IN ({placeholders})',
+ SELECT_COMMENTS_ON_CLAIMS_CLAIMID + f'WHERE comment_id IN ({placeholders})',
tuple(comment_ids)
)]
@@ -169,12 +175,6 @@ def delete_comment_by_id(conn: sqlite3.Connection, comment_id: str):
return bool(curs.rowcount)
-def hide_comment_by_id(conn: sqlite3.Connection, comment_id: str):
- with conn:
- curs = conn.execute("UPDATE OR IGNORE COMMENT SET IsHidden = TRUE WHERE CommentId = ?", (comment_id,))
- return bool(curs.rowcount)
-
-
def insert_channel(conn: sqlite3.Connection, channel_name: str, channel_id: str):
with conn:
curs = conn.execute('INSERT INTO CHANNEL(ClaimId, Name) VALUES (?, ?)', (channel_id, channel_name))
@@ -189,6 +189,26 @@ def get_channel_id_from_comment_id(conn: sqlite3.Connection, comment_id: str):
return dict(channel) if channel else {}
+def get_claim_ids_from_comment_ids(conn: sqlite3.Connection, comment_ids: list):
+ with conn:
+ cids = conn.execute(
+ f""" SELECT CommentId as comment_id, LbryClaimId AS claim_id FROM COMMENT
+ WHERE CommentId IN ({', '.join('?' for _ in comment_ids)}) """,
+ tuple(comment_ids)
+ )
+ return {row['comment_id']: row['claim_id'] for row in cids.fetchall()}
+
+
+def hide_comments_by_id(conn: sqlite3.Connection, comment_ids: list):
+ with conn:
+ curs = conn.cursor()
+ curs.executemany(
+ "UPDATE COMMENT SET IsHidden = TRUE WHERE CommentId = ?",
+ [[c] for c in comment_ids]
+ )
+ return bool(curs.rowcount)
+
+
class DatabaseWriter(object):
_writer = None
From d64044d3b258cc980ef27b75b063520c8938bf98 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 9 Aug 2019 00:50:59 -0400
Subject: [PATCH 16/21] Adds claim_search wrapper for lbrynet api
---
src/database/writes.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/database/writes.py b/src/database/writes.py
index a1cc57b..3be5475 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -57,6 +57,8 @@ async def write_comment(app, params):
async def hide_comment(app, comment_id):
return await coroutine(hide_comment_by_id)(app['writer'], comment_id)
+async def claim_search(app, **kwargs):
+ return (await request_lbrynet(app, 'claim_search', **kwargs))['items'][0]
# comment_ids: [
# {
From dc01e9b251f1a7ad2798e48e41a567b2a67f9ad0 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 9 Aug 2019 00:51:56 -0400
Subject: [PATCH 17/21] Replace singular hide_comment function with batch
hide_comments function
---
src/database/writes.py | 48 +++++++++++++++++++++++-------------------
src/server/handles.py | 8 +++----
2 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/src/database/writes.py b/src/database/writes.py
index 3be5475..16d1f17 100644
--- a/src/database/writes.py
+++ b/src/database/writes.py
@@ -3,12 +3,12 @@ import sqlite3
from asyncio import coroutine
+from src.database.queries import hide_comments_by_id
from src.database.queries import delete_comment_by_id
from src.database.queries import get_comment_or_none
from src.database.queries import insert_comment
from src.database.queries import insert_channel
-from src.database.queries import get_channel_id_from_comment_id
-from src.database.queries import hide_comment_by_id
+from src.database.queries import get_claim_ids_from_comment_ids
from src.server.misc import is_authentic_delete_signal
from src.server.misc import request_lbrynet
from src.server.misc import validate_signature_from_claim
@@ -54,28 +54,32 @@ async def write_comment(app, params):
return await coroutine(create_comment_or_error)(app['writer'], **params)
-async def hide_comment(app, comment_id):
- return await coroutine(hide_comment_by_id)(app['writer'], comment_id)
+async def hide_comments(app, comment_ids):
+ return await coroutine(hide_comments_by_id)(app['writer'], comment_ids)
+
async def claim_search(app, **kwargs):
return (await request_lbrynet(app, 'claim_search', **kwargs))['items'][0]
-# comment_ids: [
-# {
-# "comment_id": id,
-# "signing_ts": signing_ts,
-# "signature": signature
-# },
-# ...
-# ]
-async def hide_comment_if_authorized(app, comment_id, signing_ts, signature):
- channel = get_channel_id_from_comment_id(app['reader'], comment_id)
- claim = await request_lbrynet(app, 'claim_search', claim_id=channel['channel_id'])
- claim = claim['items'][0]
- if not validate_signature_from_claim(claim, signature, signing_ts, comment_id):
- raise ValueError('Invalid Signature')
- job = await app['comment_scheduler'].spawn(hide_comment(app, comment_id))
- return {
- 'hidden': await job.wait()
- }
+async def hide_comments_where_authorized(app, pieces: list):
+ comment_cids = get_claim_ids_from_comment_ids(
+ conn=app['reader'],
+ comment_ids=[p['comment_id'] for p in pieces]
+ )
+ # TODO: Amortize this process
+ claims = {}
+ comments_to_hide = []
+ for p in pieces:
+ claim_id = comment_cids[p['comment_id']]
+ if claim_id not in claims:
+ claims[claim_id] = await claim_search(app, claim_id=claim_id, no_totals=True)
+ channel = claims[claim_id].get('signing_channel')
+ if validate_signature_from_claim(channel, p['signature'], p['signing_ts'], p['comment_id']):
+ comments_to_hide.append(p['comment_id'])
+
+ if comments_to_hide:
+ job = await app['comment_scheduler'].spawn(hide_comments(app, comments_to_hide))
+ await job.wait()
+
+ return {'hidden': comments_to_hide}
diff --git a/src/server/handles.py b/src/server/handles.py
index 5951960..1f6cc44 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -16,7 +16,7 @@ from src.server.misc import is_valid_credential_input
from src.server.misc import make_error
from src.database.writes import delete_comment_if_authorized
from src.database.writes import write_comment
-from src.database.writes import hide_comment_if_authorized
+from src.database.writes import hide_comments_where_authorized
logger = logging.getLogger(__name__)
@@ -59,8 +59,8 @@ async def handle_delete_comment(app, params):
return await delete_comment_if_authorized(app, **params)
-async def handle_hide_comment(app, params):
- return await hide_comment_if_authorized(app, **params)
+async def handle_hide_comments(app, params):
+ return await hide_comments_where_authorized(app, **params)
METHODS = {
@@ -73,7 +73,7 @@ METHODS = {
'create_comment': handle_create_comment,
'delete_comment': handle_delete_comment,
'abandon_comment': handle_delete_comment,
- 'hide_comment': handle_hide_comment,
+ 'hide_comments': handle_hide_comments
}
From 3e7af6ee47e3efb525c39f9299a028fdcaa397e0 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 9 Aug 2019 00:52:34 -0400
Subject: [PATCH 18/21] Adds script to count valid signatures in the database
---
scripts/valid_signatures.py | 84 +++++++++++++++++++++++++++++++++++++
1 file changed, 84 insertions(+)
create mode 100644 scripts/valid_signatures.py
diff --git a/scripts/valid_signatures.py b/scripts/valid_signatures.py
new file mode 100644
index 0000000..cc206d8
--- /dev/null
+++ b/scripts/valid_signatures.py
@@ -0,0 +1,84 @@
+import binascii
+import hashlib
+import json
+import sqlite3
+import asyncio
+
+import aiohttp
+
+from src.server.misc import is_signature_valid, get_encoded_signature
+from src.server.database import clean
+
+
+async def request_lbrynet(url, method, **params):
+ body = {'method': method, 'params': {**params}}
+ async with aiohttp.request('POST', url, json=body) as req:
+ try:
+ resp = await req.json()
+ finally:
+ if 'result' in resp:
+ return resp['result']
+
+
+def get_comments_with_signatures(_conn: sqlite3.Connection) -> list:
+ with _conn:
+ curs = _conn.execute("SELECT * FROM COMMENTS_ON_CLAIMS WHERE signature IS NOT NULL")
+ return [dict(r) for r in curs.fetchall()]
+
+
+def is_valid_signature(pubkey, channel_id, signature, signing_ts, data: str) -> bool:
+ try:
+ if pubkey:
+ claim_hash = binascii.unhexlify(channel_id.encode())[::-1]
+ injest = b''.join((signing_ts.encode(), claim_hash, data.encode()))
+ return is_signature_valid(
+ encoded_signature=get_encoded_signature(signature),
+ signature_digest=hashlib.sha256(injest).digest(),
+ public_key_bytes=binascii.unhexlify(pubkey.encode())
+ )
+ else:
+ raise Exception("Pubkey is null")
+ except Exception as e:
+ print(e)
+ return False
+
+
+async def get_channel_pubkeys(comments: list):
+ urls = {c['channel_url'] for c in comments}
+ claims = await request_lbrynet('http://localhost:5279', 'resolve', urls=list(urls))
+ cids = {c['channel_id']: None for c in comments}
+ error_claims = []
+ for url, claim in claims.items():
+ if 'error' not in claim:
+ cids.update({
+ claim['claim_id']: claim['value']['public_key']
+ })
+ else:
+ error_claims.append({url: claim})
+ return cids, error_claims
+
+
+def count_valid_signatures(cmts: list, chan_pubkeys: dict):
+ invalid_comments = []
+ for c in cmts:
+ pubkey = chan_pubkeys.get(c['channel_id'])
+ if not is_valid_signature(pubkey, c['channel_id'], c['signature'], c['signing_ts'], c['comment']):
+ invalid_comments.append(c)
+ return len(cmts) - len(invalid_comments), invalid_comments
+
+
+if __name__ == '__main__':
+ conn = sqlite3.connect('database/default.db')
+ conn.row_factory = sqlite3.Row
+ comments = get_comments_with_signatures(conn)
+ loop = asyncio.get_event_loop()
+ chan_keys, errored = loop.run_until_complete(get_channel_pubkeys(comments))
+ valid_sigs, invalid_coms = count_valid_signatures(comments, chan_keys)
+ print(f'Total Signatures: {len(comments)}\nValid Signatures: {valid_sigs}')
+ print(f'Invalid Signatures: {len(comments) - valid_sigs}')
+ print(f'Percent Valid: {round(valid_sigs/len(comments)*100, 3)}%')
+ print(f'# Unresolving claims: {len(errored)}')
+ print(f'Num invalid comments: {len(invalid_coms)}')
+ print(json.dumps(errored, indent=2))
+ json.dump(invalid_coms, 'invalid_coms.json', indent=2)
+
From 7f9e475324e817e1f8e6738b00c79b9be05872e1 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 9 Aug 2019 00:53:10 -0400
Subject: [PATCH 19/21] `request.forwarded` -> `request.remote`
---
src/server/handles.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/server/handles.py b/src/server/handles.py
index 1f6cc44..77ea225 100644
--- a/src/server/handles.py
+++ b/src/server/handles.py
@@ -108,7 +108,7 @@ async def process_json(app, body: dict) -> dict:
@atomic
async def api_endpoint(request: web.Request):
try:
- web.access_logger.info(f'Forwarded headers: {request.forwarded}')
+ web.access_logger.info(f'Forwarded headers: {request.remote}')
body = await request.json()
if type(body) is list or type(body) is dict:
if type(body) is list:
From 332fa2bcd068157d3c01540db52d1a31025023a7 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 9 Aug 2019 01:20:00 -0400
Subject: [PATCH 20/21] Updates unittests to be compliant with batched hide
comments refactor
---
tests/database_test.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/tests/database_test.py b/tests/database_test.py
index ebf3477..adf0e4a 100644
--- a/tests/database_test.py
+++ b/tests/database_test.py
@@ -10,7 +10,7 @@ from src.database.queries import get_comment_ids
from src.database.queries import get_claim_comments
from src.database.queries import get_claim_hidden_comments
from src.database.writes import create_comment_or_error
-from src.database.queries import hide_comment_by_id
+from src.database.queries import hide_comments_by_id
from src.database.queries import delete_comment_by_id
from tests.testcase import DatabaseTestCase
@@ -199,11 +199,11 @@ class TestDatabaseOperations(DatabaseTestCase):
comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
self.assertFalse(comment['is_hidden'])
- success = hide_comment_by_id(self.conn, comm['comment_id'])
+ success = hide_comments_by_id(self.conn, [comm['comment_id']])
self.assertTrue(success)
comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
self.assertTrue(comment['is_hidden'])
- success = hide_comment_by_id(self.conn, comm['comment_id'])
+ success = hide_comments_by_id(self.conn, [comm['comment_id']])
self.assertTrue(success)
comment = get_comments_by_id(self.conn, [comm['comment_id']]).pop()
self.assertTrue(comment['is_hidden'])
@@ -211,11 +211,13 @@ class TestDatabaseOperations(DatabaseTestCase):
def test08DeleteComments(self):
comm = create_comment_or_error(self.conn, 'Comment #1', self.claimId, '1'*40, '@Doge123', 'a'*128, '123')
comments = get_claim_comments(self.conn, self.claimId)
- self.assertIn(comm, comments['items'])
+ match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items']))
+ self.assertTrue(match)
deleted = delete_comment_by_id(self.conn, comm['comment_id'])
self.assertTrue(deleted)
comments = get_claim_comments(self.conn, self.claimId)
- self.assertNotIn(comm, comments['items'])
+ match = list(filter(lambda x: comm['comment_id'] == x['comment_id'], comments['items']))
+ self.assertFalse(match)
deleted = delete_comment_by_id(self.conn, comm['comment_id'])
self.assertFalse(deleted)
@@ -261,7 +263,7 @@ class ListDatabaseTest(DatabaseTestCase):
self.assertEqual(len(comments), comment_list['total_items'])
self.assertIn('has_hidden_comments', comment_list)
self.assertFalse(comment_list['has_hidden_comments'])
- hide_comment_by_id(self.conn, comm2['comment_id'])
+ hide_comments_by_id(self.conn, [comm2['comment_id']])
default_comments = get_claim_hidden_comments(self.conn, claim_id)
self.assertIn('has_hidden_comments', default_comments)
From 6933a9110d3694e39ab141159fb5c24a472bfec8 Mon Sep 17 00:00:00 2001
From: Oleg Silkin <o.silkin98@gmail.com>
Date: Fri, 9 Aug 2019 03:12:54 -0400
Subject: [PATCH 21/21] update
---
config/conf.json | 2 +-
tests/database_test.py | 1 -
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/config/conf.json b/config/conf.json
index a41249f..4e4d027 100644
--- a/config/conf.json
+++ b/config/conf.json
@@ -1,6 +1,6 @@
{
"PATH": {
- "DATABASE": "database/comments.db",
+ "DATABASE": "database/default.db",
"ERROR_LOG": "logs/error.log",
"DEBUG_LOG": "logs/debug.log",
"SERVER_LOG": "logs/server.log"
diff --git a/tests/database_test.py b/tests/database_test.py
index adf0e4a..068e45e 100644
--- a/tests/database_test.py
+++ b/tests/database_test.py
@@ -222,7 +222,6 @@ class TestDatabaseOperations(DatabaseTestCase):
self.assertFalse(deleted)
-
class ListDatabaseTest(DatabaseTestCase):
def setUp(self) -> None:
super().setUp()