From 635aebfeeb55ac846018b4b99feb9476ab9c7d89 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sat, 27 Jun 2020 23:00:12 -0400 Subject: [PATCH] fixed null claim name support for postgres --- lbry/db/query_context.py | 19 ++++++------------- lbry/schema/url.py | 2 +- .../integration/blockchain/test_blockchain.py | 7 +++++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lbry/db/query_context.py b/lbry/db/query_context.py index da9515a47..6a1e2c9b8 100644 --- a/lbry/db/query_context.py +++ b/lbry/db/query_context.py @@ -383,31 +383,24 @@ class BulkLoader: row['claim_id'] = txo.claim_id row['claim_hash'] = txo.claim_hash try: - claim_name = txo.claim_name - if '\x00' in claim_name: - # log.error(f"Name for claim {txo.claim_id} contains a NULL (\\x00) character, skipping.") - pass - else: - row['claim_name'] = claim_name + row['claim_name'] = txo.claim_name.replace('\x00', '') except UnicodeDecodeError: - # log.error(f"Name for claim {txo.claim_id} contains invalid unicode, skipping.") pass return row def claim_to_rows(self, txo: Output) -> Tuple[dict, List]: try: - assert txo.claim_name - assert txo.normalized_name - except Exception: - #self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.") + claim_name = txo.claim_name.replace('\x00', '') + normalized_name = txo.normalized_name + except UnicodeDecodeError: return {}, [] tx = txo.tx_ref.tx claim_hash = txo.claim_hash claim_record = { 'claim_hash': claim_hash, 'claim_id': txo.claim_id, - 'claim_name': txo.claim_name, - 'normalized': txo.normalized_name, + 'claim_name': claim_name, + 'normalized': normalized_name, 'address': txo.get_address(self.ledger), 'txo_hash': txo.ref.hash, 'amount': txo.amount, diff --git a/lbry/schema/url.py b/lbry/schema/url.py index 8f68600bd..04ac62b99 100644 --- a/lbry/schema/url.py +++ b/lbry/schema/url.py @@ -44,7 +44,7 @@ URL_REGEX = _create_url_regex() def normalize_name(name): - return unicodedata.normalize('NFD', name).casefold() + return unicodedata.normalize('NFD', name).casefold().replace('\x00', '') class PathSegment(NamedTuple): diff --git a/tests/integration/blockchain/test_blockchain.py b/tests/integration/blockchain/test_blockchain.py index b2800895a..dce9fe4a2 100644 --- a/tests/integration/blockchain/test_blockchain.py +++ b/tests/integration/blockchain/test_blockchain.py @@ -11,6 +11,7 @@ from lbry import Config, Database, RegTestLedger, Transaction, Output, Input from lbry.crypto.base58 import Base58 from lbry.schema.claim import Stream, Channel from lbry.schema.support import Support +from lbry.blockchain.script import OutputScript from lbry.blockchain.lbrycrd import Lbrycrd from lbry.blockchain.sync import BlockchainSync from lbry.blockchain.dewies import dewies_to_lbc, lbc_to_dewies @@ -690,6 +691,12 @@ class TestGeneralBlockchainSync(SyncingBlockchainTestCase): claims = await search() self.assertEqual(0, len(claims)) + async def test_null_claim_name(self): + await self.create_claim(name='\x00') + await self.generate(1) + empty_name, = await self.db.search_claims() + self.assertEqual('', empty_name.normalized_name) + async def test_short_and_canonical_urls(self): search = self.db.search_claims