From 5bdd87c904d7b3f502866f66f741d7925ed3ea15 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Thu, 14 Feb 2019 12:54:23 -0700 Subject: [PATCH] removed name-to-claim lookup table added unit test for Greek and made it work pylint: revert bad move revert claim_sequence changes fixed broken test --- .gitignore | 1 + docs/cli/index.html | 6 +- docs/index.html | 6 +- lbrynet/extras/daemon/Daemon.py | 5 +- lbrynet/extras/wallet/claim_proofs.py | 5 +- lbrynet/extras/wallet/resolve.py | 14 +- .../extras/wallet/server/block_processor.py | 2 - lbrynet/extras/wallet/server/db.py | 61 ++----- lbrynet/extras/wallet/server/session.py | 171 ++++++++++-------- lbrynet/schema/uri.py | 2 +- tests/integration/test_claim_commands.py | 48 +++++ tests/integration/testcase.py | 6 +- tests/unit/schema/test_lbryschema.py | 1 - 13 files changed, 182 insertions(+), 146 deletions(-) diff --git a/.gitignore b/.gitignore index 9a32e52da..758ec3cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /.tox /.idea /.coverage +/lbry-venv lbrynet.egg-info __pycache__ diff --git a/docs/cli/index.html b/docs/cli/index.html index ba95b837f..84f637e26 100644 --- a/docs/cli/index.html +++ b/docs/cli/index.html @@ -1409,7 +1409,7 @@ Returns: 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, + 'claim_sequence': (int) claim sequence number (or -1 if unknown), 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -1954,7 +1954,6 @@ Returns: 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -1979,7 +1978,6 @@ Returns: 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -2586,4 +2584,4 @@ Returns: - \ No newline at end of file + diff --git a/docs/index.html b/docs/index.html index 911393056..a8a095c22 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1339,7 +1339,7 @@ Returns: 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, + 'claim_sequence': (int) claim sequence number (or -1 if unknown), 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -1786,7 +1786,6 @@ Returns: 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -1811,7 +1810,6 @@ Returns: 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -2312,4 +2310,4 @@ Returns: - \ No newline at end of file + diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index 36fb3bd5c..178f7ab83 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -1464,7 +1464,7 @@ class Daemon(metaclass=JSONRPCServerType): 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, + 'claim_sequence': (int) claim sequence number (or -1 if unknown), 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -1489,7 +1489,7 @@ class Daemon(metaclass=JSONRPCServerType): 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, + 'claim_sequence': (int) claim sequence number (or -1 if unknown), 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, @@ -2329,7 +2329,6 @@ class Daemon(metaclass=JSONRPCServerType): 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, - 'claim_sequence': (int) claim sequence number, 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, diff --git a/lbrynet/extras/wallet/claim_proofs.py b/lbrynet/extras/wallet/claim_proofs.py index dd1a2a29b..5b98c75ea 100644 --- a/lbrynet/extras/wallet/claim_proofs.py +++ b/lbrynet/extras/wallet/claim_proofs.py @@ -74,9 +74,10 @@ def verify_proof(proof, root_hash, name): if 'txhash' in proof and 'nOut' in proof: if not verified_value: raise InvalidProofError("mismatch between proof claim and outcome") + target = reverse_computed_name[::-1].encode('ISO-8859-1').decode() if 'txhash' in proof and 'nOut' in proof: - if name != reverse_computed_name[::-1]: + if name != target: raise InvalidProofError("name did not match proof") - if not name.startswith(reverse_computed_name[::-1]): + if not name.startswith(target): raise InvalidProofError("name fragment does not match proof") return True diff --git a/lbrynet/extras/wallet/resolve.py b/lbrynet/extras/wallet/resolve.py index 69920a0cc..544d39053 100644 --- a/lbrynet/extras/wallet/resolve.py +++ b/lbrynet/extras/wallet/resolve.py @@ -369,8 +369,14 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas return {'error': 'name is not claimed'} if 'proof' in result: + proof_name = name + if 'name' in result: + proof_name = result['name'] + name = result['name'] + if 'normalized_name' in result: + proof_name = result['normalized_name'] try: - verify_proof(result['proof'], claim_trie_root, name) + verify_proof(result['proof'], claim_trie_root, proof_name) except InvalidProofError: return {'error': "Proof was invalid"} return _parse_proof_result(name, result) @@ -421,7 +427,8 @@ def _decode_claim_result(claim): log.warning('Got an invalid claim while parsing, please report: %s', claim) claim['hex'] = None claim['value'] = None - claim['error'] = "Failed to parse: missing value" + backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else '' + claim['error'] = "Failed to parse: missing value." + backend_message return claim try: decoded = smart_decode(claim['value']) @@ -486,6 +493,7 @@ def pick_winner_from_channel_path_collision(claims_in_channel): continue if winner is None: winner = claim - elif claim['claim_sequence'] < winner['claim_sequence']: + elif claim['height'] < winner['height'] or \ + (claim['height'] == winner['height'] and claim['nout'] < winner['nout']): winner = claim return winner diff --git a/lbrynet/extras/wallet/server/block_processor.py b/lbrynet/extras/wallet/server/block_processor.py index ca7c44f9e..aeea10ee9 100644 --- a/lbrynet/extras/wallet/server/block_processor.py +++ b/lbrynet/extras/wallet/server/block_processor.py @@ -80,7 +80,6 @@ class LBRYBlockProcessor(BlockProcessor): if claim_info.cert_id: self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id) self.db.put_claim_info(claim_id, claim_info) - self.db.put_claim_for_name(claim_info.name, claim_id) self.db.put_claim_id_for_outpoint(txid, nout, claim_id) return claim_id, None @@ -116,7 +115,6 @@ class LBRYBlockProcessor(BlockProcessor): if undo_claim_info.cert_id: cert_id = self._checksig(undo_claim_info.name, undo_claim_info.value, undo_claim_info.address) self.db.put_claim_id_signed_by_cert_id(cert_id, claim_id) - self.db.put_claim_for_name(undo_claim_info.name, claim_id) self.db.put_claim_id_for_outpoint(undo_claim_info.txid, undo_claim_info.nout, claim_id) def backup_txs(self, txs): diff --git a/lbrynet/extras/wallet/server/db.py b/lbrynet/extras/wallet/server/db.py index b92bcad67..bcdffed2e 100644 --- a/lbrynet/extras/wallet/server/db.py +++ b/lbrynet/extras/wallet/server/db.py @@ -13,10 +13,9 @@ class LBRYDB(DB): def __init__(self, *args, **kwargs): self.claim_cache = {} - self.claims_for_name_cache = {} self.claims_signed_by_cert_cache = {} self.outpoint_to_claim_id_cache = {} - self.claims_db = self.names_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None + self.claims_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None # stores deletes not yet flushed to disk self.pending_abandons = {} super().__init__(*args, **kwargs) @@ -24,7 +23,6 @@ class LBRYDB(DB): def close(self): self.batched_flush_claims() self.claims_db.close() - self.names_db.close() self.signatures_db.close() self.outpoint_to_claim_id_db.close() self.claim_undo_db.close() @@ -42,12 +40,10 @@ class LBRYDB(DB): return log_reason('closing claim DBs to re-open', for_sync) self.claims_db.close() - self.names_db.close() self.signatures_db.close() self.outpoint_to_claim_id_db.close() self.claim_undo_db.close() self.claims_db = self.db_class('claims', for_sync) - self.names_db = self.db_class('names', for_sync) self.signatures_db = self.db_class('signatures', for_sync) self.outpoint_to_claim_id_db = self.db_class('outpoint_claim_id', for_sync) self.claim_undo_db = self.db_class('claim_undo', for_sync) @@ -60,21 +56,18 @@ class LBRYDB(DB): def batched_flush_claims(self): with self.claims_db.write_batch() as claims_batch: - with self.names_db.write_batch() as names_batch: - with self.signatures_db.write_batch() as signed_claims_batch: - with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch: - self.flush_claims(claims_batch, names_batch, signed_claims_batch, - outpoint_batch) + with self.signatures_db.write_batch() as signed_claims_batch: + with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch: + self.flush_claims(claims_batch, signed_claims_batch, outpoint_batch) - def flush_claims(self, batch, names_batch, signed_claims_batch, outpoint_batch): + def flush_claims(self, batch, signed_claims_batch, outpoint_batch): flush_start = time.time() - write_claim, write_name, write_cert = batch.put, names_batch.put, signed_claims_batch.put + write_claim, write_cert = batch.put, signed_claims_batch.put write_outpoint = outpoint_batch.put - delete_claim, delete_outpoint, delete_name = batch.delete, outpoint_batch.delete, names_batch.delete + delete_claim, delete_outpoint = batch.delete, outpoint_batch.delete delete_cert = signed_claims_batch.delete for claim_id, outpoints in self.pending_abandons.items(): claim = self.get_claim_info(claim_id) - self.remove_claim_for_name(claim.name, claim_id) if claim.cert_id: self.remove_claim_from_certificate_claims(claim.cert_id, claim_id) self.remove_certificate(claim_id) @@ -86,11 +79,6 @@ class LBRYDB(DB): write_claim(key, claim) else: delete_claim(key) - for name, claims in self.claims_for_name_cache.items(): - if not claims: - delete_name(name) - else: - write_name(name, msgpack.dumps(claims)) for cert_id, claims in self.claims_signed_by_cert_cache.items(): if not claims: delete_cert(cert_id) @@ -101,15 +89,13 @@ class LBRYDB(DB): write_outpoint(key, claim_id) else: delete_outpoint(key) - self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints, {:,d} names ' + self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints ' 'and {:,d} certificates added while {:,d} were abandoned in {:.1f}s, committing...' .format(self.db_height, len(self.claim_cache), len(self.outpoint_to_claim_id_cache), - len(self.claims_for_name_cache), len(self.claims_signed_by_cert_cache), len(self.pending_abandons), time.time() - flush_start)) self.claim_cache = {} - self.claims_for_name_cache = {} self.claims_signed_by_cert_cache = {} self.outpoint_to_claim_id_cache = {} self.pending_abandons = {} @@ -117,7 +103,6 @@ class LBRYDB(DB): def assert_flushed(self, flush_data): super().assert_flushed(flush_data) assert not self.claim_cache - assert not self.claims_for_name_cache assert not self.claims_signed_by_cert_cache assert not self.outpoint_to_claim_id_cache assert not self.pending_abandons @@ -142,27 +127,6 @@ class LBRYDB(DB): key = tx_hash + struct.pack('>I', tx_idx) return self.outpoint_to_claim_id_cache.get(key) or self.outpoint_to_claim_id_db.get(key) - def get_claims_for_name(self, name): - if name in self.claims_for_name_cache: - return self.claims_for_name_cache[name] - db_claims = self.names_db.get(name) - return msgpack.loads(db_claims) if db_claims else {} - - def put_claim_for_name(self, name, claim_id): - self.logger.info("[+] Adding claim {} for name {}.".format(hash_to_hex_str(claim_id), name)) - claims = self.get_claims_for_name(name) - claims.setdefault(claim_id, max(claims.values() or [0]) + 1) - self.claims_for_name_cache[name] = claims - - def remove_claim_for_name(self, name, claim_id): - self.logger.info("[-] Removing claim from name: {} - {}".format(hash_to_hex_str(claim_id), name)) - claims = self.get_claims_for_name(name) - claim_n = claims.pop(claim_id) - for _claim_id, number in claims.items(): - if number > claim_n: - claims[_claim_id] = number - 1 - self.claims_for_name_cache[name] = claims - def get_signed_claim_ids_by_cert_id(self, cert_id): if cert_id in self.claims_signed_by_cert_cache: return self.claims_signed_by_cert_cache[cert_id] @@ -170,17 +134,20 @@ class LBRYDB(DB): return msgpack.loads(db_claims, use_list=True) if db_claims else [] def put_claim_id_signed_by_cert_id(self, cert_id, claim_id): - self.logger.info("[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))) + msg = "[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)) + self.logger.info(msg) certs = self.get_signed_claim_ids_by_cert_id(cert_id) certs.append(claim_id) self.claims_signed_by_cert_cache[cert_id] = certs def remove_certificate(self, cert_id): - self.logger.info("[-] Removing certificate: {}".format(hash_to_hex_str(cert_id))) + msg = "[-] Removing certificate: {}".format(hash_to_hex_str(cert_id)) + self.logger.info(msg) self.claims_signed_by_cert_cache[cert_id] = [] def remove_claim_from_certificate_claims(self, cert_id, claim_id): - self.logger.info("[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))) + msg = "[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)) + self.logger.info(msg) certs = self.get_signed_claim_ids_by_cert_id(cert_id) if claim_id in certs: certs.remove(claim_id) diff --git a/lbrynet/extras/wallet/server/session.py b/lbrynet/extras/wallet/server/session.py index 5c4280b68..c86378185 100644 --- a/lbrynet/extras/wallet/server/session.py +++ b/lbrynet/extras/wallet/server/session.py @@ -1,4 +1,5 @@ import math +import unicodedata as uda from binascii import unhexlify, hexlify from torba.rpc.jsonrpc import RPCError @@ -6,7 +7,7 @@ from torba.server.hash import hash_to_hex_str from torba.server.session import ElectrumX from torba.server import util -from lbrynet.schema.uri import parse_lbry_uri +from lbrynet.schema.uri import parse_lbry_uri, CLAIM_ID_MAX_LENGTH from lbrynet.schema.error import URIParseError from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor from lbrynet.extras.wallet.server.db import LBRYDB @@ -131,22 +132,29 @@ class LBRYElectrumX(ElectrumX): claim_ids = self.get_claim_ids_signed_by(certificate_id) return await self.batched_formatted_claims_from_daemon(claim_ids) + def claimtrie_getclaimssignedbyidminimal(self, certificate_id): + claim_ids = self.get_claim_ids_signed_by(certificate_id) + ret = [] + for claim_id in claim_ids: + raw_claim_id = unhexlify(claim_id)[::-1] + info = self.db.get_claim_info(raw_claim_id) + if info: + ret.append({ + 'claim_id': claim_id, + 'height': info.height, + 'name': info.name.decode() + }) + return ret + def get_claim_ids_signed_by(self, certificate_id): raw_certificate_id = unhexlify(certificate_id)[::-1] raw_claim_ids = self.db.get_signed_claim_ids_by_cert_id(raw_certificate_id) return list(map(hash_to_hex_str, raw_claim_ids)) - def get_signed_claims_with_name_for_channel(self, channel_id, name): - claim_ids_for_name = list(self.db.get_claims_for_name(name.encode('ISO-8859-1')).keys()) - claim_ids_for_name = set(map(hash_to_hex_str, claim_ids_for_name)) - channel_claim_ids = set(self.get_claim_ids_signed_by(channel_id)) - return claim_ids_for_name.intersection(channel_claim_ids) - async def claimtrie_getclaimssignedbynthtoname(self, name, n): - n = int(n) - for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items(): - if n == sequence: - return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim_id)) + claim = self.claimtrie_getnthclaimforname(name, n) + if claim and 'claim_id' in claim: + return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim['claim_id'])) async def claimtrie_getclaimsintx(self, txid): # TODO: this needs further discussion. @@ -161,28 +169,27 @@ class LBRYElectrumX(ElectrumX): if proof_has_winning_claim(proof): tx_hash, nout = proof['txhash'], int(proof['nOut']) transaction_info = await self.daemon.getrawtransaction(tx_hash, True) - result['transaction'] = transaction_info['hex'] - result['height'] = (self.db.db_height - transaction_info['confirmations']) + 1 + result['transaction'] = transaction_info['hex'] # should have never included this (or the call to get it) raw_claim_id = self.db.get_claim_id_from_outpoint(unhexlify(tx_hash)[::-1], nout) - sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id) - if sequence: - claim_id = hexlify(raw_claim_id[::-1]).decode() - claim_info = await self.daemon.getclaimbyid(claim_id) - if not claim_info or not claim_info.get('value'): - claim_info = await self.slow_get_claim_by_id_using_name(claim_id) - result['claim_sequence'] = sequence - result['claim_id'] = claim_id - supports = self.format_supports_from_daemon(claim_info.get('supports', [])) # fixme: lbrycrd#124 - result['supports'] = supports - else: - self.logger.warning('tx has no claims in db: %s %s', tx_hash, nout) + claim_id = hexlify(raw_claim_id[::-1]).decode() + claim = await self.claimtrie_getclaimbyid(claim_id) + result.update(claim) + return result async def claimtrie_getnthclaimforname(self, name, n): n = int(n) - for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items(): - if n == sequence: - return await self.claimtrie_getclaimbyid(hash_to_hex_str(claim_id)) + result = await self.claimtrie_getclaimsforname(name) + if 'claims' in result and len(result['claims']) > n >= 0: + # TODO: revist this after lbrycrd_#209 to see if we can sort by claim_sequence at this point + result['claims'].sort(key=lambda c: (int(c['height']), int(c['nout']))) + result['claims'][n]['claim_sequence'] = n + return result['claims'][n] + + async def claimtrie_getpartialmatch(self, name, part): + result = await self.claimtrie_getclaimsforname(name) + if 'claims' in result: + return next(filter(lambda x: x['claim_id'].starts_with(part), result['claims']), None) async def claimtrie_getclaimsforname(self, name): claims = await self.daemon.getclaimsforname(name) @@ -198,37 +205,38 @@ class LBRYElectrumX(ElectrumX): async def batched_formatted_claims_from_daemon(self, claim_ids): claims = await self.daemon.getclaimsbyids(claim_ids) result = [] - for claim, claim_id in zip(claims, claim_ids): + for claim in claims: if claim and claim.get('value'): result.append(self.format_claim_from_daemon(claim)) - else: - recovered_claim = await self.slow_get_claim_by_id_using_name(claim_id) - if recovered_claim: - result.append(self.format_claim_from_daemon(recovered_claim)) return result def format_claim_from_daemon(self, claim, name=None): - '''Changes the returned claim data to the format expected by lbrynet and adds missing fields.''' + """Changes the returned claim data to the format expected by lbrynet and adds missing fields.""" + if not claim: return {} - name = name or claim['name'] + + # this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd + # it will be fixed after the lbrycrd upstream merge to v17 is done + # it originated as a fear of terminals not supporting unicode. alas, they all do + + if 'name' in claim: + name = claim['name'].encode('ISO-8859-1').decode() claim_id = claim['claimId'] raw_claim_id = unhexlify(claim_id)[::-1] - if not self.db.get_claim_info(raw_claim_id): - #raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id)) + info = self.db.get_claim_info(raw_claim_id) + if not info: + # raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id)) return {} - address = self.db.get_claim_info(raw_claim_id).address.decode() - sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id) - if not sequence: - return {} - supports = self.format_supports_from_daemon(claim.get('supports', [])) # fixme: lbrycrd#124 + address = info.address.decode() + supports = self.format_supports_from_daemon(claim.get('supports', [])) amount = get_from_possible_keys(claim, 'amount', 'nAmount') height = get_from_possible_keys(claim, 'height', 'nHeight') effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount') valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight') - return { + result = { "name": name, "claim_id": claim['claimId'], "txid": claim['txid'], @@ -237,12 +245,19 @@ class LBRYElectrumX(ElectrumX): "depth": self.db.db_height - height, "height": height, "value": hexlify(claim['value'].encode('ISO-8859-1')).decode(), - "claim_sequence": sequence, # from index "address": address, # from index - "supports": supports, # fixme: to be included in lbrycrd#124 + "supports": supports, "effective_amount": effective_amount, - "valid_at_height": valid_at_height # TODO PR lbrycrd to include it + "valid_at_height": valid_at_height } + if 'claim_sequence' in claim: + # TODO: ensure that lbrycrd #209 fills in this value + result['claim_sequence'] = claim['claim_sequence'] + else: + result['claim_sequence'] = -1 + if 'normalized_name' in claim: + result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode() + return result def format_supports_from_daemon(self, supports): return [[support['txid'], support['n'], get_from_possible_keys(support, 'amount', 'nAmount')] for @@ -251,8 +266,6 @@ class LBRYElectrumX(ElectrumX): async def claimtrie_getclaimbyid(self, claim_id): self.assert_claim_id(claim_id) claim = await self.daemon.getclaimbyid(claim_id) - if not claim or not claim.get('value'): - claim = await self.slow_get_claim_by_id_using_name(claim_id) return self.format_claim_from_daemon(claim) async def claimtrie_getclaimsbyids(self, *claim_ids): @@ -279,20 +292,16 @@ class LBRYElectrumX(ElectrumX): pass raise RPCError(1, f'{value} should be a claim id hash') - async def slow_get_claim_by_id_using_name(self, claim_id): - # TODO: temporary workaround for a lbrycrd bug on indexing. Should be removed when it gets stable - raw_claim_id = unhexlify(claim_id)[::-1] - claim = self.db.get_claim_info(raw_claim_id) - if claim: - name = claim.name.decode('ISO-8859-1') - claims = await self.daemon.getclaimsforname(name) - for claim in claims['claims']: - if claim['claimId'] == claim_id: - claim['name'] = name - self.logger.warning( - 'Recovered a claim missing from lbrycrd index: %s %s', name, claim_id - ) - return claim + def normalize_name(self, name): + # this is designed to match lbrycrd; change it here if it changes there + return uda.normalize('NFD', name).casefold() + + def claim_matches_name(self, claim, name): + if not name: + return False + if 'normalized_name' in claim: + return self.normalize_name(name) == claim['normalized_name'] + return name == claim['name'] async def claimtrie_getvalueforuri(self, block_hash, uri, known_certificates=None): # TODO: this thing is huge, refactor @@ -312,8 +321,11 @@ class LBRYElectrumX(ElectrumX): # TODO: this is also done on the else, refactor if parsed_uri.claim_id: - certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id) - if certificate_info and certificate_info['name'] == parsed_uri.name: + if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH: + certificate_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id) + else: + certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id) + if certificate_info and self.claim_matches_name(certificate_info, parsed_uri.name): certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info} elif parsed_uri.claim_sequence: certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence) @@ -327,26 +339,28 @@ class LBRYElectrumX(ElectrumX): if certificate and 'claim_id' not in certificate['result']: return result - if certificate and not parsed_uri.path: + if certificate: result['certificate'] = certificate channel_id = certificate['result']['claim_id'] - claims_in_channel = await self.claimtrie_getclaimssignedbyid(channel_id) - result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height']) - for claim in claims_in_channel if claim} - elif certificate: - result['certificate'] = certificate - channel_id = certificate['result']['claim_id'] - claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path) - claims = await self.batched_formatted_claims_from_daemon(claim_ids_matching_name) + claims_in_channel = self.claimtrie_getclaimssignedbyidminimal(channel_id) + if not parsed_uri.path: + result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height']) + for claim in claims_in_channel} + else: + # making an assumption that there aren't case conflicts on an existing channel + norm_path = self.normalize_name(parsed_uri.path) + result['unverified_claims_for_name'] = {claim['claim_id']: (claim['name'], claim['height']) + for claim in claims_in_channel + if self.normalize_name(claim['name']) == norm_path} - claims_in_channel = {claim['claim_id']: (claim['name'], claim['height']) - for claim in claims} - result['unverified_claims_for_name'] = claims_in_channel else: claim = None if parsed_uri.claim_id: - claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id) - if claim_info and claim_info['name'] == parsed_uri.name: + if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH: + claim_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id) + else: + claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id) + if claim_info and self.claim_matches_name(claim_info, parsed_uri.name): claim = {'resolution_type': CLAIM_ID, 'result': claim_info} elif parsed_uri.claim_sequence: claim_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence) @@ -369,6 +383,7 @@ class LBRYElectrumX(ElectrumX): 'result': certificate} result['certificate'] = certificate result['claim'] = claim + return result async def claimtrie_getvalueforuris(self, block_hash, *uris): diff --git a/lbrynet/schema/uri.py b/lbrynet/schema/uri.py index e4e415d41..cda0e70c9 100644 --- a/lbrynet/schema/uri.py +++ b/lbrynet/schema/uri.py @@ -139,7 +139,7 @@ def get_schema_regex(): protocol = _named("protocol", re.escape(PROTOCOL)) # Define basic building blocks - valid_name_char = "[a-zA-Z0-9\-]" # these characters are the only valid name characters + valid_name_char = "[^=&#:$@%?/]" # from the grammar section of https://spec.lbry.io/ name_content = valid_name_char + '+' name_min_channel_length = valid_name_char + '{' + str(CHANNEL_NAME_MIN_LENGTH) + ',}' diff --git a/tests/integration/test_claim_commands.py b/tests/integration/test_claim_commands.py index 2880562ee..667f1e332 100644 --- a/tests/integration/test_claim_commands.py +++ b/tests/integration/test_claim_commands.py @@ -371,3 +371,51 @@ class ClaimCommands(CommandTestCase): self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False) self.assertEqual(txs2[0]['value'], '0.0') self.assertEqual(txs2[0]['fee'], '-0.0001415') + + async def test_normalization_resolution(self): + + # this test assumes that the lbrycrd forks normalization at height == 250 on regtest + + c1 = await self.make_claim('ΣίσυφοςfiÆ', '0.1') + c2 = await self.make_claim('ΣΊΣΥΦΟσFIæ', '0.2') + + r1 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ') + r2 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ') + + r1c = list(r1.values())[0]['claim']['claim_id'] + r2c = list(r2.values())[0]['claim']['claim_id'] + self.assertEqual(c1['claim_id'], r1c) + self.assertEqual(c2['claim_id'], r2c) + self.assertNotEqual(r1c, r2c) + + await self.generate(50) + head = await self.daemon.jsonrpc_block_show() + self.assertTrue(head['height'] > 250) + + r3 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ') + r4 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ') + + r3c = list(r3.values())[0]['claim']['claim_id'] + r4c = list(r4.values())[0]['claim']['claim_id'] + r3n = list(r3.values())[0]['claim']['name'] + r4n = list(r4.values())[0]['claim']['name'] + + self.assertEqual(c2['claim_id'], r3c) + self.assertEqual(c2['claim_id'], r4c) + self.assertEqual(r3c, r4c) + self.assertEqual(r3n, r4n) + + +# for debugging the full stack: +# class RunFullStackForLiveTesting(CommandTestCase): +# +# VERBOSITY = logging.INFO +# +# async def asyncDaemonStart(self): +# await self.daemon.start() +# +# async def test_full_stack(self): +# self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance()) +# print("Running: do your testing now.") +# while True: +# await asyncio.sleep(0.1) \ No newline at end of file diff --git a/tests/integration/testcase.py b/tests/integration/testcase.py index 1c2bdab01..0ca6ec6ee 100644 --- a/tests/integration/testcase.py +++ b/tests/integration/testcase.py @@ -67,6 +67,9 @@ class CommandTestCase(IntegrationTestCase): MANAGER = LbryWalletManager VERBOSITY = logging.WARN + async def asyncDaemonStart(self): + await self.daemon.initialize() + async def asyncSetUp(self): await super().asyncSetUp() @@ -78,6 +81,7 @@ class CommandTestCase(IntegrationTestCase): conf.data_dir = self.wallet_node.data_path conf.wallet_dir = self.wallet_node.data_path conf.download_dir = self.wallet_node.data_path + print("WALLET_DIR =", self.wallet_node.data_path) conf.share_usage_data = False conf.use_upnp = False conf.reflect_streams = True @@ -106,7 +110,7 @@ class CommandTestCase(IntegrationTestCase): conf, skip_components=conf.components_to_skip, wallet=wallet_maker, exchange_rate_manager=ExchangeRateManagerComponent )) - await self.daemon.initialize() + await self.asyncDaemonStart() self.manager.old_db = self.daemon.storage server_tmp_dir = tempfile.mkdtemp() diff --git a/tests/unit/schema/test_lbryschema.py b/tests/unit/schema/test_lbryschema.py index ebf154876..b96ef29c5 100644 --- a/tests/unit/schema/test_lbryschema.py +++ b/tests/unit/schema/test_lbryschema.py @@ -72,7 +72,6 @@ parsed_uri_raises = [ ("lbry://test:1:1:1", URIParseError), ("whatever/lbry://test", URIParseError), ("lbry://lbry://test", URIParseError), - ("lbry://❀", URIParseError), ("lbry://@/what", URIParseError), ("lbry://abc:0x123", URIParseError), ("lbry://abc:0x123/page", URIParseError),