lbry-sdk/lbry/lbrynet/wallet/resolve.py

import logging

import asyncio

from binascii import unhexlify, hexlify
from lbrynet.utils import lru_cache_concurrent
from lbrynet.wallet.account import validate_claim_id
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
from lbrynet.schema.claim import Claim
from google.protobuf.message import DecodeError
from lbrynet.schema.url import URL
from lbrynet.wallet.claim_proofs import verify_proof, InvalidProofError

log = logging.getLogger(__name__)


class Resolver:

    def __init__(self, ledger):
        self.transaction_class = ledger.transaction_class
        self.network = ledger.network
        self.ledger = ledger

    async def resolve(self, page, page_size, *uris):
        uris = set(uris)
        try:
            for uri in uris:
                for part in URL.parse(uri).parts:
                    if part.claim_id:
                        validate_claim_id(part.claim_id)
            claim_trie_root = self.ledger.headers.claim_trie_root
            resolutions = await self.network.get_values_for_uris(self.ledger.headers.hash().decode(), *uris)
            if len(uris) > 1:
                return await self._batch_handle(resolutions, uris, page, page_size, claim_trie_root)
            return await self._handle_resolutions(resolutions, uris, page, page_size, claim_trie_root)
        except ValueError as err:
            return {'error': err.args[0]}
        except Exception as e:
            log.exception(e)
            return {'error': str(e)}

    async def _batch_handle(self, resolutions, uris, page, page_size, claim_trie_root):
        futs = []
        for uri in uris:
            futs.append(
                asyncio.ensure_future(self._handle_resolutions(resolutions, [uri], page, page_size, claim_trie_root))
            )
        results = await asyncio.gather(*futs)
        return dict(list(map(lambda result: list(result.items())[0], results)))

    @lru_cache_concurrent(256)
    async def _fetch_tx(self, txid):
        return self.transaction_class(unhexlify(await self.network.get_transaction(txid)))

    async def _handle_resolutions(self, resolutions, requested_uris, page, page_size, claim_trie_root):
        results = {}
        for uri in requested_uris:
            resolution = (resolutions or {}).get(uri, {})
            if resolution:
                try:
                    results[uri] = _handle_claim_result(
                        await self._handle_resolve_uri_response(uri, resolution, claim_trie_root, page, page_size),
                        uri
                    )
                except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
                    log.exception(err)
                    results[uri] = {'error': str(err)}
            else:
                results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
        return results

    async def _handle_resolve_uri_response(self, uri, resolution, claim_trie_root, page=0, page_size=10):
        result = {}
        parsed_uri = URL.parse(uri)
        certificate_response = None
        # parse an included certificate
        if 'certificate' in resolution:
            certificate_response = resolution['certificate']['result']
            certificate_resolution_type = resolution['certificate']['resolution_type']
            if certificate_resolution_type == "winning" and certificate_response:
                if 'height' in certificate_response:
                    certificate_response = _verify_proof(parsed_uri.stream.name,
                                                       claim_trie_root,
                                                       certificate_response,
                                                       ledger=self.ledger)
            elif certificate_resolution_type not in ['winning', 'claim_id', 'sequence']:
                raise Exception(f"unknown response type: {certificate_resolution_type}")
            result['certificate'] = await self.parse_and_validate_claim_result(certificate_response)
            result['claims_in_channel'] = len(resolution.get('unverified_claims_in_channel', []))

        # if this was a resolution for a name, parse the result
        if 'claim' in resolution:
            claim_response = resolution['claim']['result']
            claim_resolution_type = resolution['claim']['resolution_type']
            if claim_resolution_type == "winning" and claim_response:
                if 'height' in claim_response:
                    claim_response = _verify_proof(parsed_uri.stream.name,
                                                   claim_trie_root,
                                                   claim_response,
                                                   ledger=self.ledger)
            elif claim_resolution_type not in ["sequence", "winning", "claim_id"]:
                raise Exception(f"unknown response type: {claim_resolution_type}")
            result['claim'] = await self.parse_and_validate_claim_result(claim_response,
                                                                         certificate_response)

        # if this was a resolution for a name in a channel make sure there is only one valid
        # match
        elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
            unverified_claims_for_name = resolution['unverified_claims_for_name']

            channel_info = await self.get_channel_claims_page(unverified_claims_for_name,
                                                              result['certificate'], page=1)
            claims_in_channel, upper_bound = channel_info

            if not claims_in_channel:
                log.error("No valid claims for this name for this channel")
            elif len(claims_in_channel) > 1:
                log.warning("Multiple signed claims for the same name.")
                winner = pick_winner_from_channel_path_collision(claims_in_channel)
                if winner:
                    result['claim'] = winner
                else:
                    log.error("No valid claims for this name for this channel")
            else:
                result['claim'] = claims_in_channel[0]

        # parse and validate claims in a channel iteratively into pages of results
        elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
            ids_to_check = resolution['unverified_claims_in_channel']
            channel_info = await self.get_channel_claims_page(ids_to_check, result['certificate'],
                                                              page=page, page_size=page_size)
            claims_in_channel, upper_bound = channel_info

            if claims_in_channel:
                result['total_claims'] = upper_bound
                result['claims_in_channel'] = claims_in_channel
        elif 'error' not in result:
            return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}

        # invalid signatures can only return outside a channel
        if result.get('claim', {}).get('has_signature', False):
            if parsed_uri.has_stream and not result['claim']['signature_is_valid']:
                return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
        return result

    async def parse_and_validate_claim_result(self, claim_result, certificate=None):
        if not claim_result or 'value' not in claim_result:
            return claim_result
        claim_result = _decode_claim_result(claim_result)
        if claim_result.get('height'):
            claim_result['timestamp'] = self.ledger.headers[claim_result['height']]['timestamp']
        if claim_result.get('depth'):
            claim_result['confirmations'] = claim_result.pop('depth')

        if claim_result['value']:
            claim_result['has_signature'] = False
            if claim_result['value'].is_signed:
                claim_result['has_signature'] = True
                claim_tx = await self._fetch_tx(claim_result['txid'])
                if certificate is None:
                    log.info("fetching certificate to check claim signature")
                    channel_id = claim_result['value'].signing_channel_id
                    certificate = (await self.network.get_claims_by_ids([channel_id])).get(channel_id)
                    if not certificate:
                        log.warning('Certificate %s not found', channel_id)
                claim_result['channel_name'] = certificate['name'] if certificate else None
                cert_tx = await self._fetch_tx(certificate['txid']) if certificate else None
                claim_result['signature_is_valid'] = validate_claim_signature_and_get_channel_name(
                    claim_result, certificate, self.ledger, claim_tx=claim_tx, cert_tx=cert_tx
                )
                # fixme: workaround while json encoder isnt used here
                if cert_tx:
                    channel_txo = cert_tx.outputs[certificate['nout']]
                    claim_result['signing_channel'] = {
                        'name': channel_txo.claim_name,
                        'claim_id': channel_txo.claim_id,
                        'value': channel_txo.claim
                    }
                    claim_result['is_channel_signature_valid'] = claim_result['signature_is_valid']

        if 'amount' in claim_result:
            claim_result['amount'] = dewies_to_lbc(claim_result['amount'])
            claim_result['effective_amount'] = dewies_to_lbc(claim_result['effective_amount'])
            claim_result['supports'] = [
                {'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
                for (txid, nout, amount) in claim_result['supports']
            ]

        claim_result['height'] = claim_result.get('height', -1) or -1
        claim_result['permanent_url'] = f"lbry://{claim_result['name']}#{claim_result['claim_id']}"

        return claim_result

    @staticmethod
    def prepare_claim_queries(start_position, query_size, channel_claim_infos):
        queries = [tuple()]
        names = {}
        # a table of index counts for the sorted claim ids, including ignored claims
        absolute_position_index = {}

        block_sorted_infos = sorted(channel_claim_infos.items(), key=lambda x: int(x[1][1]))
        per_block_infos = {}
        for claim_id, (name, height) in block_sorted_infos:
            claims = per_block_infos.get(height, [])
            claims.append((claim_id, name))
            per_block_infos[height] = sorted(claims, key=lambda x: int(x[0], 16))

        abs_position = 0

        for height in sorted(per_block_infos.keys(), reverse=True):
            for claim_id, name in per_block_infos[height]:
                names[claim_id] = name
                absolute_position_index[claim_id] = abs_position
                if abs_position >= start_position:
                    if len(queries[-1]) >= query_size:
                        queries.append(tuple())
                    queries[-1] += (claim_id,)
                abs_position += 1
        return queries, names, absolute_position_index

    async def iter_channel_claims_pages(self, queries, claim_positions, claim_names, certificate,
                                  page_size=10):
        # lbryum server returns a dict of {claim_id: (name, claim_height)}
        # first, sort the claims by block height (and by claim id int value within a block).

        # map the sorted claims into getclaimsbyids queries of query_size claim ids each

        # send the batched queries to lbryum server and iteratively validate and parse
        # the results, yield a page of results at a time.

        # these results can include those where `signature_is_valid` is False. if they are skipped,
        # page indexing becomes tricky, as the number of results isn't known until after having
        # processed them.
        # TODO: fix ^ in lbrynet.schema

        async def iter_validate_channel_claims():
            formatted_claims = []
            for claim_ids in queries:
                batch_result = await self.network.get_claims_by_ids(claim_ids)
                for claim_id in claim_ids:
                    claim = batch_result[claim_id]
                    if claim['name'] == claim_names[claim_id]:
                        formatted_claim = await self.parse_and_validate_claim_result(claim, certificate)
                        formatted_claim['absolute_channel_position'] = claim_positions[
                            claim['claim_id']]
                        formatted_claims.append(formatted_claim)
                    else:
                        log.warning("ignoring claim with name mismatch %s %s", claim['name'],
                                    claim['claim_id'])
                return formatted_claims

        results = []

        for claim in await iter_validate_channel_claims():
            results.append(claim)

            # if there is a full page of results, yield it
            if len(results) and len(results) % page_size == 0:
                return results[-page_size:]

        return results

    async def get_channel_claims_page(self, channel_claim_infos, certificate, page, page_size=10):
        page = page or 0
        page_size = max(page_size, 1)
        if page_size > 500:
            raise Exception("page size above maximum allowed")
        start_position = (page - 1) * page_size
        queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
                                                                     channel_claim_infos)
        upper_bound = len(claim_positions)
        if not page:
            return None, upper_bound
        if start_position > upper_bound:
            raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
        page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
                                                              certificate, page_size=page_size)
        return page_generator, upper_bound


def _verify_proof(name, claim_trie_root, result, ledger):
    """
    Verify proof for name claim
    """
    support_amount = sum([amt for (stxid, snout, amt) in result['supports']])

    def _build_response(name, tx, nOut):
        output = tx.outputs[nOut]
        r = {
            'name': name,
            'value': hexlify(output.script.values['claim']),
            'claim_id': output.claim_id,
            'txid': tx.id,
            'nout': nOut,
            'amount': output.amount,
            'effective_amount': output.amount + support_amount,
            'height': result['height'],
            'confirmations': result['depth'],
            'claim_sequence': result['claim_sequence'],
            'address': output.get_address(ledger),
            'valid_at_height': result['valid_at_height'],
            'timestamp': ledger.headers[result['height']]['timestamp'],
            'supports': result['supports']
        }
        return r

    def _parse_proof_result(name, result):
        if 'txhash' in result['proof'] and 'nOut' in result['proof']:
            if 'transaction' in result:
                tx = ledger.transaction_class(raw=unhexlify(result['transaction']))
                nOut = result['proof']['nOut']
                if result['proof']['txhash'] == tx.id:
                    if 0 <= nOut < len(tx.outputs):
                        if tx.outputs[nOut].claim_name == name:
                            return _build_response(name, tx, nOut)
                        return {'error': 'name in proof did not match requested name'}
                    outputs = len(tx['outputs'])
                    return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
                return {'error': "computed txid did not match given transaction: %s vs %s" %
                                 (tx.id, result['proof']['txhash'])
                        }
            return {'error': "didn't receive a transaction with the proof"}
        return {'error': 'name is not claimed'}

    if 'proof' in result:
        name = result.get('name', name)
        proof_name = result.get('normalized_name', name)
        try:
            verify_proof(result['proof'], claim_trie_root, proof_name)
        except InvalidProofError:
            return {'error': "Proof was invalid"}
        return _parse_proof_result(name, result)
    else:
        return {'error': "proof not in result"}


def validate_claim_signature_and_get_channel_name(claim_result, certificate_claim, ledger,
                                                  claim_tx=None, cert_tx=None):
    valid_signature = False
    if cert_tx and certificate_claim and claim_tx and claim_result:
        valid_signature = claim_tx.outputs[claim_result['nout']].is_signed_by(
            cert_tx.outputs[certificate_claim['nout']], ledger
        )
        if not valid_signature:
            log.warning("lbry://%s#%s has an invalid signature",
                        claim_result['name'], claim_result['claim_id'])
    return valid_signature


# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
def _decode_claim_result(claim):
    if 'decoded_claim' in claim:
        return claim
    if 'value' not in claim:
        log.warning('Got an invalid claim while parsing, please report: %s', claim)
        claim['protobuf'] = None
        claim['value'] = None
        backend_message = ' SDK message: ' + claim.get('error', '')
        claim['error'] = "Failed to parse: missing value." + backend_message
        return claim
    try:
        if not isinstance(claim['value'], Claim):
            claim['value'] = Claim.from_bytes(unhexlify(claim['value']))
        claim['protobuf'] = hexlify(claim['value'].to_bytes())
        claim['decoded_claim'] = True
    except DecodeError:
        claim['decoded_claim'] = False
        claim['protobuf'] = claim['value']
        claim['value'] = None
    return claim


def _handle_claim_result(results, uri):
    if not results:
        raise UnknownURI(uri)

    if 'error' in results:
        if results['error'] in ['name is not claimed', 'claim not found']:
            if 'claim_id' in results:
                raise UnknownClaimID(results['claim_id'])
            if 'name' in results:
                raise UnknownNameError(results['name'])
            if 'uri' in results:
                raise UnknownURI(results['uri'])
            if 'outpoint' in results:
                raise UnknownOutpoint(results['outpoint'])
        raise Exception(results['error'])
    if not {'value', 'claim', 'certificate'}.intersection(results.keys()):
        raise Exception(f'result in unexpected format:{results}')
    return results


def pick_winner_from_channel_path_collision(claims_in_channel):
    # we should be doing this by effective amount so we pick the controlling claim, however changing the resolved
    # claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
    # stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
    # potentially triggering that.
    winner = {}
    for claim in claims_in_channel:
        if not winner or claim['height'] < winner['height'] or \
                (claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
            winner = claim if claim['signature_is_valid'] else winner
    return winner or None