From f83289f87632a3751dd6fb12af6f3aaf8d4efee8 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 5 Nov 2020 18:16:21 -0300 Subject: [PATCH] add owner column, apply censor --- lbry/blockchain/sync/claims.py | 8 +++++--- lbry/db/queries/search.py | 8 ++++++-- lbry/db/tables.py | 1 + lbry/schema/result.py | 20 +++++++------------ .../integration/blockchain/test_blockchain.py | 3 +++ 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lbry/blockchain/sync/claims.py b/lbry/blockchain/sync/claims.py index 1a6add799..095fd5aaf 100644 --- a/lbry/blockchain/sync/claims.py +++ b/lbry/blockchain/sync/claims.py @@ -287,11 +287,13 @@ def update_channel_stats(blocks: Tuple[int, int], initial_sync: int, p: Progress def update_claim_filters(blocking_channel_hashes, filtering_channel_hashes, p: ProgressContext): def select_reposts(channel_hashes, filter_type=0): return select( - Claim.c.reposted_claim_hash, filter_type).where( + Claim.c.reposted_claim_hash, filter_type, Claim.c.channel_hash).where( (Claim.c.channel_hash.in_(filtering_channel_hashes)) & (Claim.c.reposted_claim_hash.isnot(None))) p.ctx.execute(ClaimFilter.delete()) + # order matters: first we insert the blocked ones. Then the filtered ones. + # If there is already a block in place, that takes priority because a block is just a harder filter p.ctx.execute(ClaimFilter.insert().from_select( - ['claim_hash', 'filter_type'], select_reposts(blocking_channel_hashes, 1))) + ['claim_hash', 'filter_type', 'owner_channel_hash'], select_reposts(blocking_channel_hashes, 2))) p.ctx.execute(p.ctx.insert_or_ignore(ClaimFilter).from_select( - ['claim_hash', 'filter_type'], select_reposts(filtering_channel_hashes, 0))) + ['claim_hash', 'filter_type', 'owner_channel_hash'], select_reposts(filtering_channel_hashes, 1))) diff --git a/lbry/db/queries/search.py b/lbry/db/queries/search.py index 4a8b136b8..d598576fe 100644 --- a/lbry/db/queries/search.py +++ b/lbry/db/queries/search.py @@ -14,7 +14,7 @@ from lbry.blockchain.transaction import Output from ..utils import query from ..query_context import context -from ..tables import TX, TXO, Claim, Support, Trending +from ..tables import TX, TXO, Claim, Support, Trending, ClaimFilter from ..constants import ( TXO_TYPES, STREAM_TYPES, ATTRIBUTE_ARRAY_MAX_LENGTH, SEARCH_INTEGER_PARAMS, SEARCH_ORDER_FIELDS @@ -126,7 +126,9 @@ BASE_SELECT_CLAIM_COLUMNS = BASE_SELECT_TXO_COLUMNS + [ func.coalesce(Trending.c.trending_local, 0).label('trending_local'), func.coalesce(Trending.c.trending_mixed, 0).label('trending_mixed'), func.coalesce(Trending.c.trending_global, 0).label('trending_global'), - func.coalesce(Trending.c.trending_group, 0).label('trending_group') + func.coalesce(Trending.c.trending_group, 0).label('trending_group'), + func.coalesce(ClaimFilter.c.filter_type, 0).label('censor_type'), + ClaimFilter.c.owner_channel_hash.label('censor_owner_hash') ] @@ -276,6 +278,7 @@ def select_claims(cols: List = None, for_count=False, **constraints) -> Select: .select_from( Claim.join(TXO).join(TX).join(Trending, Trending.c.claim_hash == Claim.c.claim_hash, isouter=True) .join(channel_claim, Claim.c.channel_hash == channel_claim.c.claim_hash, isouter=True) + .join(ClaimFilter, ClaimFilter.c.claim_hash == Claim.c.claim_hash, isouter=True) ), **constraints ) @@ -306,6 +309,7 @@ def search_claims(**constraints) -> Tuple[List[Output], Optional[int], Optional[ return [], total, search_censor rows = ctx.fetchall(select_claims(**constraints)) + rows = search_censor.apply(rows) txos = rows_to_txos(rows, include_tx=False) annotate_with_channels(txos) return txos, total, search_censor diff --git a/lbry/db/tables.py b/lbry/db/tables.py index f764d85bb..54557b91b 100644 --- a/lbry/db/tables.py +++ b/lbry/db/tables.py @@ -338,5 +338,6 @@ Trending = Table( ClaimFilter = Table( 'claim_filter', metadata, Column('claim_hash', LargeBinary, primary_key=True), + Column('owner_channel_hash', LargeBinary), Column('filter_type', SmallInteger), ) diff --git a/lbry/schema/result.py b/lbry/schema/result.py index 87a165411..2276ca38a 100644 --- a/lbry/schema/result.py +++ b/lbry/schema/result.py @@ -32,21 +32,15 @@ class Censor: self.censored = {} self.total = 0 + def apply(self, rows): + return [row for row in rows if not self.censor(row)] + def censor(self, row) -> bool: - was_censored = False - for claim_hash, lookup in ( - (row['claim_hash'], self.streams), - (row['claim_hash'], self.channels), - (row['channel_hash'], self.channels), - (row['reposted_claim_hash'], self.streams), - (row['reposted_claim_hash'], self.channels)): - censoring_channel_hash = lookup.get(claim_hash) - if censoring_channel_hash: - was_censored = True - self.censored.setdefault(censoring_channel_hash, 0) - self.censored[censoring_channel_hash] += 1 - break + was_censored = row['censor_type'] > 0 if was_censored: + censoring_channel_hash = row['censor_owner_hash'] + self.censored.setdefault(censoring_channel_hash, 0) + self.censored[censoring_channel_hash] += 1 self.total += 1 return was_censored diff --git a/tests/integration/blockchain/test_blockchain.py b/tests/integration/blockchain/test_blockchain.py index 45d851dab..89a07c987 100644 --- a/tests/integration/blockchain/test_blockchain.py +++ b/tests/integration/blockchain/test_blockchain.py @@ -1392,6 +1392,9 @@ class TestClaimtrieSync(SyncingBlockchainTestCase): await self.create_claim(sign=moderator_chan, name="blocking_bad", repost=bad_content.claim_id) self.sync.filtering_channel_hashes.add(moderator_chan.claim_hash) await self.generate(1) + results = await self.db.search_claims(channel="@some_channel") + self.assertEqual(len(results.rows), 1) + self.assertEqual(results.censor.censored.get(moderator_chan.claim_hash), 1) @skip