removed name-to-claim lookup table

added unit test for Greek and made it work


pylint: revert bad move


revert claim_sequence changes


fixed broken test

.gitignore

@@ -3,6 +3,7 @@
 /.tox
 /.idea
 /.coverage
+/lbry-venv
 
 lbrynet.egg-info
 __pycache__

docs/cli/index.html

@@ -1409,7 +1409,7 @@ Returns:
                     'amount': (float) claim amount,
                     'effective_amount': (float) claim amount including supports,
                     'claim_id': (str) claim id,
-                    'claim_sequence': (int) claim sequence number,
+                    'claim_sequence': (int) claim sequence number (or -1 if unknown),
                     'decoded_claim': (bool) whether or not the claim value was decoded,
                     'height': (int) claim height,
                     'depth': (int) claim depth,
@@ -1954,7 +1954,6 @@ Returns:
                 'amount': (float) claim amount,
                 'effective_amount': (float) claim amount including supports,
                 'claim_id': (str) claim id,
-                'claim_sequence': (int) claim sequence number,
                 'decoded_claim': (bool) whether or not the claim value was decoded,
                 'height': (int) claim height,
                 'depth': (int) claim depth,
@@ -1979,7 +1978,6 @@ Returns:
                 'amount': (float) claim amount,
                 'effective_amount': (float) claim amount including supports,
                 'claim_id': (str) claim id,
-                'claim_sequence': (int) claim sequence number,
                 'decoded_claim': (bool) whether or not the claim value was decoded,
                 'height': (int) claim height,
                 'depth': (int) claim depth,

docs/index.html

@@ -1339,7 +1339,7 @@ Returns:
                     'amount': (float) claim amount,
                     'effective_amount': (float) claim amount including supports,
                     'claim_id': (str) claim id,
-                    'claim_sequence': (int) claim sequence number,
+                    'claim_sequence': (int) claim sequence number (or -1 if unknown),
                     'decoded_claim': (bool) whether or not the claim value was decoded,
                     'height': (int) claim height,
                     'depth': (int) claim depth,
@@ -1786,7 +1786,6 @@ Returns:
                 'amount': (float) claim amount,
                 'effective_amount': (float) claim amount including supports,
                 'claim_id': (str) claim id,
-                'claim_sequence': (int) claim sequence number,
                 'decoded_claim': (bool) whether or not the claim value was decoded,
                 'height': (int) claim height,
                 'depth': (int) claim depth,
@@ -1811,7 +1810,6 @@ Returns:
                 'amount': (float) claim amount,
                 'effective_amount': (float) claim amount including supports,
                 'claim_id': (str) claim id,
-                'claim_sequence': (int) claim sequence number,
                 'decoded_claim': (bool) whether or not the claim value was decoded,
                 'height': (int) claim height,
                 'depth': (int) claim depth,

lbrynet/extras/daemon/Daemon.py

@@ -1464,7 +1464,7 @@ class Daemon(metaclass=JSONRPCServerType):
                         'amount': (float) claim amount,
                         'effective_amount': (float) claim amount including supports,
                         'claim_id': (str) claim id,
-                        'claim_sequence': (int) claim sequence number,
+                        'claim_sequence': (int) claim sequence number (or -1 if unknown),
                         'decoded_claim': (bool) whether or not the claim value was decoded,
                         'height': (int) claim height,
                         'depth': (int) claim depth,
@@ -1489,7 +1489,7 @@ class Daemon(metaclass=JSONRPCServerType):
                         'amount': (float) claim amount,
                         'effective_amount': (float) claim amount including supports,
                         'claim_id': (str) claim id,
-                        'claim_sequence': (int) claim sequence number,
+                        'claim_sequence': (int) claim sequence number (or -1 if unknown),
                         'decoded_claim': (bool) whether or not the claim value was decoded,
                         'height': (int) claim height,
                         'depth': (int) claim depth,
@@ -2329,7 +2329,6 @@ class Daemon(metaclass=JSONRPCServerType):
                             'amount': (float) claim amount,
                             'effective_amount': (float) claim amount including supports,
                             'claim_id': (str) claim id,
-                            'claim_sequence': (int) claim sequence number,
                             'decoded_claim': (bool) whether or not the claim value was decoded,
                             'height': (int) claim height,
                             'depth': (int) claim depth,

lbrynet/extras/wallet/claim_proofs.py

@@ -74,9 +74,10 @@ def verify_proof(proof, root_hash, name):
     if 'txhash' in proof and 'nOut' in proof:
         if not verified_value:
             raise InvalidProofError("mismatch between proof claim and outcome")
+    target = reverse_computed_name[::-1].encode('ISO-8859-1').decode()
     if 'txhash' in proof and 'nOut' in proof:
-        if name != reverse_computed_name[::-1]:
+        if name != target:
             raise InvalidProofError("name did not match proof")
-    if not name.startswith(reverse_computed_name[::-1]):
+    if not name.startswith(target):
         raise InvalidProofError("name fragment does not match proof")
     return True

lbrynet/extras/wallet/resolve.py

@@ -369,8 +369,14 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
         return {'error': 'name is not claimed'}
 
     if 'proof' in result:
+        proof_name = name
+        if 'name' in result:
+            proof_name = result['name']
+            name = result['name']
+        if 'normalized_name' in result:
+            proof_name = result['normalized_name']
         try:
-            verify_proof(result['proof'], claim_trie_root, name)
+            verify_proof(result['proof'], claim_trie_root, proof_name)
         except InvalidProofError:
             return {'error': "Proof was invalid"}
         return _parse_proof_result(name, result)
@@ -421,7 +427,8 @@ def _decode_claim_result(claim):
         log.warning('Got an invalid claim while parsing, please report: %s', claim)
         claim['hex'] = None
         claim['value'] = None
-        claim['error'] = "Failed to parse: missing value"
+        backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else ''
+        claim['error'] = "Failed to parse: missing value." + backend_message
         return claim
     try:
         decoded = smart_decode(claim['value'])
@@ -486,6 +493,7 @@ def pick_winner_from_channel_path_collision(claims_in_channel):
             continue
         if winner is None:
             winner = claim
-        elif claim['claim_sequence'] < winner['claim_sequence']:
+        elif claim['height'] < winner['height'] or \
+                (claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
             winner = claim
     return winner

lbrynet/extras/wallet/server/block_processor.py

@@ -80,7 +80,6 @@ class LBRYBlockProcessor(BlockProcessor):
         if claim_info.cert_id:
             self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
         self.db.put_claim_info(claim_id, claim_info)
-        self.db.put_claim_for_name(claim_info.name, claim_id)
         self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
         return claim_id, None
 
@@ -116,7 +115,6 @@ class LBRYBlockProcessor(BlockProcessor):
             if undo_claim_info.cert_id:
                 cert_id = self._checksig(undo_claim_info.name, undo_claim_info.value, undo_claim_info.address)
                 self.db.put_claim_id_signed_by_cert_id(cert_id, claim_id)
-            self.db.put_claim_for_name(undo_claim_info.name, claim_id)
             self.db.put_claim_id_for_outpoint(undo_claim_info.txid, undo_claim_info.nout, claim_id)
 
     def backup_txs(self, txs):

lbrynet/extras/wallet/server/db.py

@@ -13,10 +13,9 @@ class LBRYDB(DB):
 
     def __init__(self, *args, **kwargs):
         self.claim_cache = {}
-        self.claims_for_name_cache = {}
         self.claims_signed_by_cert_cache = {}
         self.outpoint_to_claim_id_cache = {}
-        self.claims_db = self.names_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
+        self.claims_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
         # stores deletes not yet flushed to disk
         self.pending_abandons = {}
         super().__init__(*args, **kwargs)
@@ -24,7 +23,6 @@ class LBRYDB(DB):
     def close(self):
         self.batched_flush_claims()
         self.claims_db.close()
-        self.names_db.close()
         self.signatures_db.close()
         self.outpoint_to_claim_id_db.close()
         self.claim_undo_db.close()
@@ -42,12 +40,10 @@ class LBRYDB(DB):
                 return
             log_reason('closing claim DBs to re-open', for_sync)
             self.claims_db.close()
-            self.names_db.close()
             self.signatures_db.close()
             self.outpoint_to_claim_id_db.close()
             self.claim_undo_db.close()
         self.claims_db = self.db_class('claims', for_sync)
-        self.names_db = self.db_class('names', for_sync)
         self.signatures_db = self.db_class('signatures', for_sync)
         self.outpoint_to_claim_id_db = self.db_class('outpoint_claim_id', for_sync)
         self.claim_undo_db = self.db_class('claim_undo', for_sync)
@@ -60,21 +56,18 @@ class LBRYDB(DB):
 
     def batched_flush_claims(self):
         with self.claims_db.write_batch() as claims_batch:
-            with self.names_db.write_batch() as names_batch:
             with self.signatures_db.write_batch() as signed_claims_batch:
                 with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
-                        self.flush_claims(claims_batch, names_batch, signed_claims_batch,
+                    self.flush_claims(claims_batch, signed_claims_batch, outpoint_batch)
-                                          outpoint_batch)
 
-    def flush_claims(self, batch, names_batch, signed_claims_batch, outpoint_batch):
+    def flush_claims(self, batch, signed_claims_batch, outpoint_batch):
         flush_start = time.time()
-        write_claim, write_name, write_cert = batch.put, names_batch.put, signed_claims_batch.put
+        write_claim, write_cert = batch.put, signed_claims_batch.put
         write_outpoint = outpoint_batch.put
-        delete_claim, delete_outpoint, delete_name = batch.delete, outpoint_batch.delete, names_batch.delete
+        delete_claim, delete_outpoint = batch.delete, outpoint_batch.delete
         delete_cert = signed_claims_batch.delete
         for claim_id, outpoints in self.pending_abandons.items():
             claim = self.get_claim_info(claim_id)
-            self.remove_claim_for_name(claim.name, claim_id)
             if claim.cert_id:
                 self.remove_claim_from_certificate_claims(claim.cert_id, claim_id)
             self.remove_certificate(claim_id)
@@ -86,11 +79,6 @@ class LBRYDB(DB):
                 write_claim(key, claim)
             else:
                 delete_claim(key)
-        for name, claims in self.claims_for_name_cache.items():
-            if not claims:
-                delete_name(name)
-            else:
-                write_name(name, msgpack.dumps(claims))
         for cert_id, claims in self.claims_signed_by_cert_cache.items():
             if not claims:
                 delete_cert(cert_id)
@@ -101,15 +89,13 @@ class LBRYDB(DB):
                 write_outpoint(key, claim_id)
             else:
                 delete_outpoint(key)
-        self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints, {:,d} names '
+        self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints '
                          'and {:,d} certificates added while {:,d} were abandoned in {:.1f}s, committing...'
                          .format(self.db_height,
                                  len(self.claim_cache), len(self.outpoint_to_claim_id_cache),
-                                 len(self.claims_for_name_cache),
                                  len(self.claims_signed_by_cert_cache), len(self.pending_abandons),
                                  time.time() - flush_start))
         self.claim_cache = {}
-        self.claims_for_name_cache = {}
         self.claims_signed_by_cert_cache = {}
         self.outpoint_to_claim_id_cache = {}
         self.pending_abandons = {}
@@ -117,7 +103,6 @@ class LBRYDB(DB):
     def assert_flushed(self, flush_data):
         super().assert_flushed(flush_data)
         assert not self.claim_cache
-        assert not self.claims_for_name_cache
         assert not self.claims_signed_by_cert_cache
         assert not self.outpoint_to_claim_id_cache
         assert not self.pending_abandons
@@ -142,27 +127,6 @@ class LBRYDB(DB):
         key = tx_hash + struct.pack('>I', tx_idx)
         return self.outpoint_to_claim_id_cache.get(key) or self.outpoint_to_claim_id_db.get(key)
 
-    def get_claims_for_name(self, name):
-        if name in self.claims_for_name_cache:
-            return self.claims_for_name_cache[name]
-        db_claims = self.names_db.get(name)
-        return msgpack.loads(db_claims) if db_claims else {}
-
-    def put_claim_for_name(self, name, claim_id):
-        self.logger.info("[+] Adding claim {} for name {}.".format(hash_to_hex_str(claim_id), name))
-        claims = self.get_claims_for_name(name)
-        claims.setdefault(claim_id, max(claims.values() or [0]) + 1)
-        self.claims_for_name_cache[name] = claims
-
-    def remove_claim_for_name(self, name, claim_id):
-        self.logger.info("[-] Removing claim from name: {} - {}".format(hash_to_hex_str(claim_id), name))
-        claims = self.get_claims_for_name(name)
-        claim_n = claims.pop(claim_id)
-        for _claim_id, number in claims.items():
-            if number > claim_n:
-                claims[_claim_id] = number - 1
-        self.claims_for_name_cache[name] = claims
-
     def get_signed_claim_ids_by_cert_id(self, cert_id):
         if cert_id in self.claims_signed_by_cert_cache:
             return self.claims_signed_by_cert_cache[cert_id]
@@ -170,17 +134,20 @@ class LBRYDB(DB):
         return msgpack.loads(db_claims, use_list=True) if db_claims else []
 
     def put_claim_id_signed_by_cert_id(self, cert_id, claim_id):
-        self.logger.info("[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
+        msg = "[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
+        self.logger.info(msg)
         certs = self.get_signed_claim_ids_by_cert_id(cert_id)
         certs.append(claim_id)
         self.claims_signed_by_cert_cache[cert_id] = certs
 
     def remove_certificate(self, cert_id):
-        self.logger.info("[-] Removing certificate: {}".format(hash_to_hex_str(cert_id)))
+        msg = "[-] Removing certificate: {}".format(hash_to_hex_str(cert_id))
+        self.logger.info(msg)
         self.claims_signed_by_cert_cache[cert_id] = []
 
     def remove_claim_from_certificate_claims(self, cert_id, claim_id):
-        self.logger.info("[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
+        msg = "[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
+        self.logger.info(msg)
         certs = self.get_signed_claim_ids_by_cert_id(cert_id)
         if claim_id in certs:
             certs.remove(claim_id)

lbrynet/extras/wallet/server/session.py

@@ -1,4 +1,5 @@
 import math
+import unicodedata as uda
 from binascii import unhexlify, hexlify
 
 from torba.rpc.jsonrpc import RPCError
@@ -6,7 +7,7 @@ from torba.server.hash import hash_to_hex_str
 from torba.server.session import ElectrumX
 from torba.server import util
 
-from lbrynet.schema.uri import parse_lbry_uri
+from lbrynet.schema.uri import parse_lbry_uri, CLAIM_ID_MAX_LENGTH
 from lbrynet.schema.error import URIParseError
 from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor
 from lbrynet.extras.wallet.server.db import LBRYDB
@@ -131,22 +132,29 @@ class LBRYElectrumX(ElectrumX):
         claim_ids = self.get_claim_ids_signed_by(certificate_id)
         return await self.batched_formatted_claims_from_daemon(claim_ids)
 
+    def claimtrie_getclaimssignedbyidminimal(self, certificate_id):
+        claim_ids = self.get_claim_ids_signed_by(certificate_id)
+        ret = []
+        for claim_id in claim_ids:
+            raw_claim_id = unhexlify(claim_id)[::-1]
+            info = self.db.get_claim_info(raw_claim_id)
+            if info:
+                ret.append({
+                    'claim_id': claim_id,
+                    'height': info.height,
+                    'name': info.name.decode()
+                })
+        return ret
+
     def get_claim_ids_signed_by(self, certificate_id):
         raw_certificate_id = unhexlify(certificate_id)[::-1]
         raw_claim_ids = self.db.get_signed_claim_ids_by_cert_id(raw_certificate_id)
         return list(map(hash_to_hex_str, raw_claim_ids))
 
-    def get_signed_claims_with_name_for_channel(self, channel_id, name):
-        claim_ids_for_name = list(self.db.get_claims_for_name(name.encode('ISO-8859-1')).keys())
-        claim_ids_for_name = set(map(hash_to_hex_str, claim_ids_for_name))
-        channel_claim_ids = set(self.get_claim_ids_signed_by(channel_id))
-        return claim_ids_for_name.intersection(channel_claim_ids)
-
     async def claimtrie_getclaimssignedbynthtoname(self, name, n):
-        n = int(n)
+        claim = self.claimtrie_getnthclaimforname(name, n)
-        for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
+        if claim and 'claim_id' in claim:
-            if n == sequence:
+            return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim['claim_id']))
-                return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim_id))
 
     async def claimtrie_getclaimsintx(self, txid):
         # TODO: this needs further discussion.
@@ -161,28 +169,27 @@ class LBRYElectrumX(ElectrumX):
         if proof_has_winning_claim(proof):
             tx_hash, nout = proof['txhash'], int(proof['nOut'])
             transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
-            result['transaction'] = transaction_info['hex']
+            result['transaction'] = transaction_info['hex']  # should have never included this (or the call to get it)
-            result['height'] = (self.db.db_height - transaction_info['confirmations']) + 1
             raw_claim_id = self.db.get_claim_id_from_outpoint(unhexlify(tx_hash)[::-1], nout)
-            sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
-            if sequence:
             claim_id = hexlify(raw_claim_id[::-1]).decode()
-                claim_info = await self.daemon.getclaimbyid(claim_id)
+            claim = await self.claimtrie_getclaimbyid(claim_id)
-                if not claim_info or not claim_info.get('value'):
+            result.update(claim)
-                    claim_info = await self.slow_get_claim_by_id_using_name(claim_id)
+
-                result['claim_sequence'] = sequence
-                result['claim_id'] = claim_id
-                supports = self.format_supports_from_daemon(claim_info.get('supports', []))  # fixme: lbrycrd#124
-                result['supports'] = supports
-            else:
-                self.logger.warning('tx has no claims in db: %s %s', tx_hash, nout)
         return result
 
     async def claimtrie_getnthclaimforname(self, name, n):
         n = int(n)
-        for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
+        result = await self.claimtrie_getclaimsforname(name)
-            if n == sequence:
+        if 'claims' in result and len(result['claims']) > n >= 0:
-                return await self.claimtrie_getclaimbyid(hash_to_hex_str(claim_id))
+            # TODO: revist this after lbrycrd_#209 to see if we can sort by claim_sequence at this point
+            result['claims'].sort(key=lambda c: (int(c['height']), int(c['nout'])))
+            result['claims'][n]['claim_sequence'] = n
+            return result['claims'][n]
+
+    async def claimtrie_getpartialmatch(self, name, part):
+        result = await self.claimtrie_getclaimsforname(name)
+        if 'claims' in result:
+            return next(filter(lambda x: x['claim_id'].starts_with(part), result['claims']), None)
 
     async def claimtrie_getclaimsforname(self, name):
         claims = await self.daemon.getclaimsforname(name)
@@ -198,37 +205,38 @@ class LBRYElectrumX(ElectrumX):
     async def batched_formatted_claims_from_daemon(self, claim_ids):
         claims = await self.daemon.getclaimsbyids(claim_ids)
         result = []
-        for claim, claim_id in zip(claims, claim_ids):
+        for claim in claims:
             if claim and claim.get('value'):
                 result.append(self.format_claim_from_daemon(claim))
-            else:
-                recovered_claim = await self.slow_get_claim_by_id_using_name(claim_id)
-                if recovered_claim:
-                    result.append(self.format_claim_from_daemon(recovered_claim))
         return result
 
     def format_claim_from_daemon(self, claim, name=None):
-        '''Changes the returned claim data to the format expected by lbrynet and adds missing fields.'''
+        """Changes the returned claim data to the format expected by lbrynet and adds missing fields."""
+
         if not claim:
             return {}
-        name = name or claim['name']
+
+        # this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd
+        # it will be fixed after the lbrycrd upstream merge to v17 is done
+        # it originated as a fear of terminals not supporting unicode. alas, they all do
+
+        if 'name' in claim:
+            name = claim['name'].encode('ISO-8859-1').decode()
         claim_id = claim['claimId']
         raw_claim_id = unhexlify(claim_id)[::-1]
-        if not self.db.get_claim_info(raw_claim_id):
+        info = self.db.get_claim_info(raw_claim_id)
-            #raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
+        if not info:
+            #  raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
             return {}
-        address = self.db.get_claim_info(raw_claim_id).address.decode()
+        address = info.address.decode()
-        sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
+        supports = self.format_supports_from_daemon(claim.get('supports', []))
-        if not sequence:
-            return {}
-        supports = self.format_supports_from_daemon(claim.get('supports', []))  # fixme: lbrycrd#124
 
         amount = get_from_possible_keys(claim, 'amount', 'nAmount')
         height = get_from_possible_keys(claim, 'height', 'nHeight')
         effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount')
         valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight')
 
-        return {
+        result = {
             "name": name,
             "claim_id": claim['claimId'],
             "txid": claim['txid'],
@@ -237,12 +245,19 @@ class LBRYElectrumX(ElectrumX):
             "depth": self.db.db_height - height,
             "height": height,
             "value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
-            "claim_sequence": sequence,  # from index
             "address": address,  # from index
-            "supports": supports, # fixme: to be included in lbrycrd#124
+            "supports": supports,
             "effective_amount": effective_amount,
-            "valid_at_height": valid_at_height  # TODO PR lbrycrd to include it
+            "valid_at_height": valid_at_height
         }
+        if 'claim_sequence' in claim:
+            # TODO: ensure that lbrycrd #209 fills in this value
+            result['claim_sequence'] = claim['claim_sequence']
+        else:
+            result['claim_sequence'] = -1
+        if 'normalized_name' in claim:
+            result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode()
+        return result
 
     def format_supports_from_daemon(self, supports):
         return [[support['txid'], support['n'], get_from_possible_keys(support, 'amount', 'nAmount')] for
@@ -251,8 +266,6 @@ class LBRYElectrumX(ElectrumX):
     async def claimtrie_getclaimbyid(self, claim_id):
         self.assert_claim_id(claim_id)
         claim = await self.daemon.getclaimbyid(claim_id)
-        if not claim or not claim.get('value'):
-            claim = await self.slow_get_claim_by_id_using_name(claim_id)
         return self.format_claim_from_daemon(claim)
 
     async def claimtrie_getclaimsbyids(self, *claim_ids):
@@ -279,20 +292,16 @@ class LBRYElectrumX(ElectrumX):
             pass
         raise RPCError(1, f'{value} should be a claim id hash')
 
-    async def slow_get_claim_by_id_using_name(self, claim_id):
+    def normalize_name(self, name):
-        # TODO: temporary workaround for a lbrycrd bug on indexing. Should be removed when it gets stable
+        # this is designed to match lbrycrd; change it here if it changes there
-        raw_claim_id = unhexlify(claim_id)[::-1]
+        return uda.normalize('NFD', name).casefold()
-        claim = self.db.get_claim_info(raw_claim_id)
+
-        if claim:
+    def claim_matches_name(self, claim, name):
-            name = claim.name.decode('ISO-8859-1')
+        if not name:
-            claims = await self.daemon.getclaimsforname(name)
+            return False
-            for claim in claims['claims']:
+        if 'normalized_name' in claim:
-                if claim['claimId'] == claim_id:
+            return self.normalize_name(name) == claim['normalized_name']
-                    claim['name'] = name
+        return name == claim['name']
-                    self.logger.warning(
-                        'Recovered a claim missing from lbrycrd index: %s %s', name, claim_id
-                    )
-                    return claim
 
     async def claimtrie_getvalueforuri(self, block_hash, uri, known_certificates=None):
         # TODO: this thing is huge, refactor
@@ -312,8 +321,11 @@ class LBRYElectrumX(ElectrumX):
 
             # TODO: this is also done on the else, refactor
             if parsed_uri.claim_id:
+                if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
+                    certificate_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
+                else:
                     certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
-                if certificate_info and certificate_info['name'] == parsed_uri.name:
+                if certificate_info and self.claim_matches_name(certificate_info, parsed_uri.name):
                     certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info}
             elif parsed_uri.claim_sequence:
                 certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
@@ -327,26 +339,28 @@ class LBRYElectrumX(ElectrumX):
             if certificate and 'claim_id' not in certificate['result']:
                 return result
 
-            if certificate and not parsed_uri.path:
+            if certificate:
                 result['certificate'] = certificate
                 channel_id = certificate['result']['claim_id']
-                claims_in_channel = await self.claimtrie_getclaimssignedbyid(channel_id)
+                claims_in_channel = self.claimtrie_getclaimssignedbyidminimal(channel_id)
+                if not parsed_uri.path:
                     result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
-                                                          for claim in claims_in_channel if claim}
+                                                              for claim in claims_in_channel}
-            elif certificate:
+                else:
-                result['certificate'] = certificate
+                    # making an assumption that there aren't case conflicts on an existing channel
-                channel_id = certificate['result']['claim_id']
+                    norm_path = self.normalize_name(parsed_uri.path)
-                claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path)
+                    result['unverified_claims_for_name'] = {claim['claim_id']: (claim['name'], claim['height'])
-                claims = await self.batched_formatted_claims_from_daemon(claim_ids_matching_name)
+                                                            for claim in claims_in_channel
+                                                            if self.normalize_name(claim['name']) == norm_path}
 
-                claims_in_channel = {claim['claim_id']: (claim['name'], claim['height'])
-                                     for claim in claims}
-                result['unverified_claims_for_name'] = claims_in_channel
         else:
             claim = None
             if parsed_uri.claim_id:
+                if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
+                    claim_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
+                else:
                     claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
-                if claim_info and claim_info['name'] == parsed_uri.name:
+                if claim_info and self.claim_matches_name(claim_info, parsed_uri.name):
                     claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
             elif parsed_uri.claim_sequence:
                 claim_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
@@ -369,6 +383,7 @@ class LBRYElectrumX(ElectrumX):
                                        'result': certificate}
                         result['certificate'] = certificate
                 result['claim'] = claim
+
         return result
 
     async def claimtrie_getvalueforuris(self, block_hash, *uris):

lbrynet/schema/uri.py

@@ -139,7 +139,7 @@ def get_schema_regex():
     protocol = _named("protocol", re.escape(PROTOCOL))
 
     # Define basic building blocks
-    valid_name_char = "[a-zA-Z0-9\-]"  # these characters are the only valid name characters
+    valid_name_char = "[^=&#:$@%?/]"  # from the grammar section of https://spec.lbry.io/
     name_content = valid_name_char + '+'
     name_min_channel_length = valid_name_char + '{' + str(CHANNEL_NAME_MIN_LENGTH) + ',}'
 

tests/integration/test_claim_commands.py

@@ -371,3 +371,51 @@ class ClaimCommands(CommandTestCase):
         self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False)
         self.assertEqual(txs2[0]['value'], '0.0')
         self.assertEqual(txs2[0]['fee'], '-0.0001415')
+
+    async def test_normalization_resolution(self):
+
+        # this test assumes that the lbrycrd forks normalization at height == 250 on regtest
+
+        c1 = await self.make_claim('ΣίσυφοςfiÆ', '0.1')
+        c2 = await self.make_claim('ΣΊΣΥΦΟσFIæ', '0.2')
+
+        r1 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
+        r2 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
+
+        r1c = list(r1.values())[0]['claim']['claim_id']
+        r2c = list(r2.values())[0]['claim']['claim_id']
+        self.assertEqual(c1['claim_id'], r1c)
+        self.assertEqual(c2['claim_id'], r2c)
+        self.assertNotEqual(r1c, r2c)
+
+        await self.generate(50)
+        head = await self.daemon.jsonrpc_block_show()
+        self.assertTrue(head['height'] > 250)
+
+        r3 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
+        r4 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
+
+        r3c = list(r3.values())[0]['claim']['claim_id']
+        r4c = list(r4.values())[0]['claim']['claim_id']
+        r3n = list(r3.values())[0]['claim']['name']
+        r4n = list(r4.values())[0]['claim']['name']
+
+        self.assertEqual(c2['claim_id'], r3c)
+        self.assertEqual(c2['claim_id'], r4c)
+        self.assertEqual(r3c, r4c)
+        self.assertEqual(r3n, r4n)
+
+
+# for debugging the full stack:
+# class RunFullStackForLiveTesting(CommandTestCase):
+#
+#     VERBOSITY = logging.INFO
+#
+#     async def asyncDaemonStart(self):
+#         await self.daemon.start()
+#
+#     async def test_full_stack(self):
+#         self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
+#         print("Running: do your testing now.")
+#         while True:
+#             await asyncio.sleep(0.1)

tests/integration/testcase.py

@@ -67,6 +67,9 @@ class CommandTestCase(IntegrationTestCase):
     MANAGER = LbryWalletManager
     VERBOSITY = logging.WARN
 
+    async def asyncDaemonStart(self):
+        await self.daemon.initialize()
+
     async def asyncSetUp(self):
         await super().asyncSetUp()
 
@@ -78,6 +81,7 @@ class CommandTestCase(IntegrationTestCase):
         conf.data_dir = self.wallet_node.data_path
         conf.wallet_dir = self.wallet_node.data_path
         conf.download_dir = self.wallet_node.data_path
+        print("WALLET_DIR =", self.wallet_node.data_path)
         conf.share_usage_data = False
         conf.use_upnp = False
         conf.reflect_streams = True
@@ -106,7 +110,7 @@ class CommandTestCase(IntegrationTestCase):
             conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
             exchange_rate_manager=ExchangeRateManagerComponent
         ))
-        await self.daemon.initialize()
+        await self.asyncDaemonStart()
         self.manager.old_db = self.daemon.storage
 
         server_tmp_dir = tempfile.mkdtemp()

tests/unit/schema/test_lbryschema.py

@@ -72,7 +72,6 @@ parsed_uri_raises = [
     ("lbry://test:1:1:1", URIParseError),
     ("whatever/lbry://test", URIParseError),
     ("lbry://lbry://test", URIParseError),
-    ("lbry://❀", URIParseError),
     ("lbry://@/what", URIParseError),
     ("lbry://abc:0x123", URIParseError),
     ("lbry://abc:0x123/page", URIParseError),