From 9d9916548b0e73e6a204cb0be97a0fb71ec51a86 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 8 Oct 2018 10:41:07 -0400 Subject: [PATCH] added channel/signature annotation to all TXOs, support for pagination --- lbrynet/daemon/Daemon.py | 123 ++++++++++++++---- lbrynet/daemon/json_response_encoder.py | 18 +-- lbrynet/wallet/account.py | 6 +- lbrynet/wallet/certificate.py | 5 - lbrynet/wallet/database.py | 54 +++++--- lbrynet/wallet/manager.py | 19 ++- lbrynet/wallet/transaction.py | 11 +- tests/integration/wallet/test_commands.py | 4 +- tests/integration/wallet/test_transactions.py | 2 +- tests/unit/wallet/test_ledger.py | 4 +- tests/unit/wallet/test_transaction.py | 2 +- 11 files changed, 170 insertions(+), 78 deletions(-) delete mode 100644 lbrynet/wallet/certificate.py diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 461b390f8..e3c004c37 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -1166,7 +1166,7 @@ class Daemon(AuthJSONRPCServer): @requires("wallet") @defer.inlineCallbacks - def jsonrpc_account_balance(self, account_id=None, address=None, include_unconfirmed=False): + def jsonrpc_account_balance(self, account_id=None, confirmations=0): """ Return the balance of an account @@ -1174,21 +1174,16 @@ class Daemon(AuthJSONRPCServer): account_balance [] [
| --address=
] [--include_unconfirmed] Options: - --account_id= : (str) If provided only the balance for this - account will be given - --address=
: (str) If provided only the balance for this - address will be given - --include_unconfirmed : (bool) Include unconfirmed + --account_id= : (str) If provided only the balance for this + account will be given. Otherwise default account. + --confirmations= : (int) Only include transactions with this many + confirmed blocks. Returns: (decimal) amount of lbry credits in wallet """ - if address is not None: - raise NotImplementedError("Limiting by address needs to be re-implemented in new wallet.") account = self.get_account_or_default(account_id) - dewies = yield account.get_balance( - 0 if include_unconfirmed else 6 - ) + dewies = yield account.get_balance(confirmations=confirmations) return dewies_to_lbc(dewies) @requires("wallet") @@ -1513,7 +1508,7 @@ class Daemon(AuthJSONRPCServer): ) @requires(WALLET_COMPONENT) - def jsonrpc_address_list(self, account_id=None): + def jsonrpc_address_list(self, account_id=None, offset=None, limit=None): """ List account addresses @@ -1522,11 +1517,26 @@ class Daemon(AuthJSONRPCServer): Options: --account_id= : (str) id of the account to use + --offset= : (int) slice address list starting at offset + --limit= : (int) limit number of addresses returned Returns: List of wallet addresses """ - return self.get_account_or_default(account_id).get_addresses() + account = self.get_account_or_default(account_id) + if None not in (offset, limit): + constraints = { + 'account': account, + 'offset': offset, + 'limit': limit + } + return { + "list": self.ledger.db.get_addresses(**constraints), + "size": self.ledger.db.get_addresses_count(**constraints), + "offset": offset, + "limit": limit + } + return account.get_addresses() @requires(WALLET_COMPONENT) def jsonrpc_address_unused(self, account_id=None): @@ -2033,16 +2043,16 @@ class Daemon(AuthJSONRPCServer): nout = 0 txo = tx.outputs[nout] log.info("Claimed a new channel! lbry://%s txid: %s nout: %d", channel_name, tx.id, nout) - defer.returnValue({ + return { "success": True, "tx": tx, "claim_id": txo.claim_id, - "claim_address": self.ledger.hash160_to_address(txo.script.values['pubkey_hash']), + "claim_address": txo.get_address(self.ledger), "output": txo - }) + } @requires(WALLET_COMPONENT) - def jsonrpc_channel_list(self): + def jsonrpc_channel_list(self, account_id=None, offset=None, limit=None): """ Get certificate claim infos for channels that can be published to @@ -2050,13 +2060,28 @@ class Daemon(AuthJSONRPCServer): channel_list Options: - None + --account_id= : (str) id of the account to use + --offset= : (int) slice channel list starting at offset + --limit= : (int) limit number of channels returned Returns: (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim is in the wallet. """ - return self.wallet_manager.channel_list() + account = self.get_account_or_default(account_id) + if None not in (offset, limit): + constraints = { + 'account': account, + 'offset': offset, + 'limit': limit + } + return { + "list": self.ledger.db.get_channels(**constraints), + "size": self.ledger.db.get_channels_count(**constraints), + "offset": offset, + "limit": limit + } + return account.get_channels() @requires(WALLET_COMPONENT) @defer.inlineCallbacks @@ -2450,9 +2475,8 @@ class Daemon(AuthJSONRPCServer): claim_id, address, self.get_dewies_or_error("amount", amount) if amount else None ) - # TODO: claim_list_mine should be merged into claim_list, but idk how to authenticate it -Grin @requires(WALLET_COMPONENT) - def jsonrpc_claim_list_mine(self, account_id=None): + def jsonrpc_claim_list_mine(self, account_id=None, offset=None, limit=None): """ List my name claims @@ -2461,6 +2485,8 @@ class Daemon(AuthJSONRPCServer): Options: --account_id= : (str) id of the account to query + --offset= : (int) slice claim list starting at offset + --limit= : (int) limit number of claims returned Returns: (list) List of name claims owned by user @@ -2484,7 +2510,20 @@ class Daemon(AuthJSONRPCServer): }, ] """ - return self.get_account_or_default(account_id).get_claims() + account = self.get_account_or_default(account_id) + if None not in (offset, limit): + constraints = { + 'account': account, + 'offset': offset, + 'limit': limit + } + return { + "list": self.ledger.db.get_claims(**constraints), + "size": self.ledger.db.get_claims_count(**constraints), + "offset": offset, + "limit": limit + } + return account.get_claims() @requires(WALLET_COMPONENT) @defer.inlineCallbacks @@ -2616,7 +2655,8 @@ class Daemon(AuthJSONRPCServer): return response @requires(WALLET_COMPONENT) - def jsonrpc_transaction_list(self, account_id=None): + @defer.inlineCallbacks + def jsonrpc_transaction_list(self, account_id=None, offset=None, limit=None): """ List transactions belonging to wallet @@ -2625,6 +2665,8 @@ class Daemon(AuthJSONRPCServer): Options: --account_id= : (str) id of the account to query + --offset= : (int) slice transaction list starting at offset + --limit= : (int) limit number of transactions returned Returns: (list) List of transactions @@ -2672,7 +2714,21 @@ class Daemon(AuthJSONRPCServer): } """ - return self.wallet_manager.get_history(self.get_account_or_default(account_id)) + account = self.get_account_or_default(account_id) + if None not in (offset, limit): + constraints = { + 'offset': offset, + 'limit': limit + } + return { + "list": self.wallet_manager.get_history( + account=account, **constraints), + "size": self.ledger.db.get_transactions_count( + account=account, **constraints), + "offset": offset, + "limit": limit + } + return self.wallet_manager.get_history(account) @requires(WALLET_COMPONENT) def jsonrpc_transaction_show(self, txid): @@ -2691,7 +2747,7 @@ class Daemon(AuthJSONRPCServer): return self.wallet_manager.get_transaction(txid) @requires(WALLET_COMPONENT) - def jsonrpc_utxo_list(self, account_id=None): + def jsonrpc_utxo_list(self, account_id=None, offset=None, limit=None): """ List unspent transaction outputs @@ -2700,6 +2756,8 @@ class Daemon(AuthJSONRPCServer): Options: --account_id= : (str) id of the account to query + --offset= : (int) slice utxo list starting at offset + --limit= : (int) limit number of utxo returned Returns: (list) List of unspent transaction outputs (UTXOs) @@ -2718,7 +2776,20 @@ class Daemon(AuthJSONRPCServer): ... ] """ - return self.get_account_or_default(account_id).get_unspent_outputs() + account = self.get_account_or_default(account_id) + if None not in (offset, limit): + constraints = { + 'account': account, + 'offset': offset, + 'limit': limit + } + return { + "list": self.ledger.db.get_utxos(**constraints), + "size": self.ledger.db.get_utxo_count(**constraints), + "offset": offset, + "limit": limit + } + return account.get_utxos() @requires(WALLET_COMPONENT) def jsonrpc_block_show(self, blockhash=None, height=None): diff --git a/lbrynet/daemon/json_response_encoder.py b/lbrynet/daemon/json_response_encoder.py index 740545ebd..b82c26d5d 100644 --- a/lbrynet/daemon/json_response_encoder.py +++ b/lbrynet/daemon/json_response_encoder.py @@ -48,6 +48,7 @@ class JSONResponseEncoder(JSONEncoder): output['is_change'] = txo.is_change if txo.is_my_account is not None: output['is_mine'] = txo.is_my_account + if txo.script.is_claim_involved: output.update({ 'name': txo.claim_name, @@ -57,18 +58,19 @@ class JSONResponseEncoder(JSONEncoder): 'is_support': txo.script.is_support_claim, 'is_update': txo.script.is_update_claim }) + + if txo.script.is_claim_name or txo.script.is_update_claim: + output['value'] = txo.claim.claim_dict + if txo.claim_name.startswith('@'): + output['has_signature'] = txo.has_signature + if txo.script.is_claim_name: - output.update({ - 'category': 'claim', - 'value': txo.claim.claim_dict - }) + output['category'] = 'claim' elif txo.script.is_update_claim: - output.update({ - 'category': 'update', - 'value': txo.claim.claim_dict - }) + output['category'] = 'update' elif txo.script.is_support_claim: output['category'] = 'support' + return output def encode_input(self, txi): diff --git a/lbrynet/wallet/account.py b/lbrynet/wallet/account.py index 6c50cd0a1..382c9c274 100644 --- a/lbrynet/wallet/account.py +++ b/lbrynet/wallet/account.py @@ -120,13 +120,13 @@ class Account(BaseAccount): constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0}) return super().get_balance(confirmations, **constraints) - def get_unspent_outputs(self, include_claims=False, **constraints): + def get_utxos(self, include_claims=False, **constraints): if not include_claims: constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0}) - return super().get_unspent_outputs(**constraints) + return super().get_utxos(**constraints) def get_channels(self): - return super().get_unspent_outputs( + return super().get_utxos( claim_type__any={'is_claim': 1, 'is_update': 1}, claim_name__like='@%' ) diff --git a/lbrynet/wallet/certificate.py b/lbrynet/wallet/certificate.py deleted file mode 100644 index 042132e39..000000000 --- a/lbrynet/wallet/certificate.py +++ /dev/null @@ -1,5 +0,0 @@ -from collections import namedtuple - - -class Certificate(namedtuple('Certificate', ('channel', 'private_key'))): - pass diff --git a/lbrynet/wallet/database.py b/lbrynet/wallet/database.py index c410cb1be..ff219056d 100644 --- a/lbrynet/wallet/database.py +++ b/lbrynet/wallet/database.py @@ -1,6 +1,5 @@ from twisted.internet import defer from torba.basedatabase import BaseDatabase -from .certificate import Certificate class WalletDatabase(BaseDatabase): @@ -48,21 +47,24 @@ class WalletDatabase(BaseDatabase): @defer.inlineCallbacks def get_txos(self, **constraints): - txos = yield super().get_txos(**constraints) - my_account = constraints.get('my_account', constraints.get('account')) + my_account = constraints.get('my_account', constraints.get('account', None)) - claim_ids = set() + txos = yield super().get_txos(**constraints) + + channel_ids = set() for txo in txos: if txo.script.is_claim_name or txo.script.is_update_claim: if 'publisherSignature' in txo.claim_dict: - claim_ids.add(txo.claim_dict['publisherSignature']['certificateId']) + channel_ids.add(txo.claim_dict['publisherSignature']['certificateId']) + if txo.claim_name.startswith('@') and my_account is not None: + txo.signature = my_account.get_certificate_private_key(txo.ref) - if claim_ids: + if channel_ids: channels = { txo.claim_id: txo for txo in (yield super().get_utxos( my_account=my_account, - claim_id__in=claim_ids + claim_id__in=channel_ids )) } for txo in txos: @@ -72,27 +74,45 @@ class WalletDatabase(BaseDatabase): return txos - def get_claims(self, **constraints): + @staticmethod + def constrain_claims(constraints): constraints['claim_type__any'] = {'is_claim': 1, 'is_update': 1} + + def get_claims(self, **constraints): + self.constrain_claims(constraints) return self.get_utxos(**constraints) - def get_channels(self, **constraints): + def get_claims_count(self, **constraints): + self.constrain_claims(constraints) + return self.get_utxo_count(**constraints) + + @staticmethod + def constrain_channels(constraints): if 'claim_name' not in constraints or 'claim_id' not in constraints: constraints['claim_name__like'] = '@%' + + def get_channels(self, **constraints): + self.constrain_channels(constraints) return self.get_claims(**constraints) + def get_channels_count(self, **constraints): + self.constrain_channels(constraints) + return self.get_claims_count(**constraints) + @defer.inlineCallbacks def get_certificates(self, private_key_accounts, exclude_without_key=False, **constraints): channels = yield self.get_channels(**constraints) certificates = [] if private_key_accounts is not None: for channel in channels: - private_key = None - for account in private_key_accounts: - private_key = account.get_certificate_private_key(channel.ref) - if private_key is not None: - break - if private_key is None and exclude_without_key: - continue - certificates.append(Certificate(channel, private_key)) + if not channel.has_signature: + private_key = None + for account in private_key_accounts: + private_key = account.get_certificate_private_key(channel.ref) + if private_key is not None: + break + if private_key is None and exclude_without_key: + continue + channel.signature = private_key + certificates.append(channel) return certificates diff --git a/lbrynet/wallet/manager.py b/lbrynet/wallet/manager.py index b1bde6d35..27c02a46d 100644 --- a/lbrynet/wallet/manager.py +++ b/lbrynet/wallet/manager.py @@ -240,9 +240,9 @@ class LbryWalletManager(BaseWalletManager): @defer.inlineCallbacks def address_is_mine(self, unknown_address, account): - for my_address in (yield account.get_addresses()): - if unknown_address == my_address: - return True + match = yield self.ledger.db.get_address(address=unknown_address, account=account) + if match is not None: + return True return False def get_transaction(self, txid: str): @@ -250,9 +250,9 @@ class LbryWalletManager(BaseWalletManager): @staticmethod @defer.inlineCallbacks - def get_history(account: BaseAccount): + def get_history(account: BaseAccount, **constraints): headers = account.ledger.headers - txs: List[Transaction] = (yield account.get_transactions()) + txs = (yield account.get_transactions(account=account, **constraints)) history = [] for tx in txs: ts = headers[tx.height]['timestamp'] @@ -301,7 +301,7 @@ class LbryWalletManager(BaseWalletManager): @staticmethod def get_utxos(account: BaseAccount): - return account.get_unspent_outputs() + return account.get_utxos() @defer.inlineCallbacks def claim_name(self, name, amount, claim_dict, certificate=None, claim_address=None): @@ -311,9 +311,9 @@ class LbryWalletManager(BaseWalletManager): claim_address = yield account.receiving.get_or_create_usable_address() if certificate: claim = claim.sign( - certificate.private_key, claim_address, certificate.channel.claim_id + certificate.signature, claim_address, certificate.claim_id ) - existing_claims = yield account.get_unspent_outputs(include_claims=True, claim_name=name) + existing_claims = yield account.get_utxos(include_claims=True, claim_name=name) if len(existing_claims) == 0: tx = yield Transaction.claim( name, claim, amount, claim_address, [account], account @@ -382,9 +382,6 @@ class LbryWalletManager(BaseWalletManager): # TODO: release reserved tx outputs in case anything fails by this point defer.returnValue(tx) - def channel_list(self): - return self.default_account.get_channels() - def get_certificates(self, private_key_accounts, exclude_without_key=True, **constraints): return self.db.get_certificates( private_key_accounts=private_key_accounts, diff --git a/lbrynet/wallet/transaction.py b/lbrynet/wallet/transaction.py index a50c572ea..4e0a25da2 100644 --- a/lbrynet/wallet/transaction.py +++ b/lbrynet/wallet/transaction.py @@ -19,16 +19,19 @@ class Output(BaseOutput): script: OutputScript script_class = OutputScript - __slots__ = '_claim_dict', 'channel' + __slots__ = '_claim_dict', 'channel', 'signature' - def __init__(self, *args, channel: Optional['Output'] = None, **kwargs) -> None: + def __init__(self, *args, channel: Optional['Output'] = None, + signature: Optional[str] = None, **kwargs) -> None: super().__init__(*args, **kwargs) self._claim_dict = None self.channel = channel + self.signature = signature def update_annotations(self, annotated): super().update_annotations(annotated) self.channel = annotated.channel if annotated else None + self.signature = annotated.signature if annotated else None def get_fee(self, ledger): name_fee = 0 @@ -76,6 +79,10 @@ class Output(BaseOutput): return "{}#{}".format(self.claim_name, self.claim_id) raise ValueError('No claim associated.') + @property + def has_signature(self): + return self.signature is not None + @classmethod def pay_claim_name_pubkey_hash( cls, amount: int, claim_name: str, claim: bytes, pubkey_hash: bytes) -> 'Output': diff --git a/tests/integration/wallet/test_commands.py b/tests/integration/wallet/test_commands.py index 27fb92416..a4b504407 100644 --- a/tests/integration/wallet/test_commands.py +++ b/tests/integration/wallet/test_commands.py @@ -113,7 +113,7 @@ class CommandTestCase(IntegrationTestCase): lbry_conf.settings.node_id = None await d2f(self.account.ensure_address_gap()) - address = (await d2f(self.account.receiving.get_addresses(1, only_usable=True)))[0] + address = (await d2f(self.account.receiving.get_addresses(limit=1, only_usable=True)))[0] sendtxid = await self.blockchain.send_to_address(address, 10) await self.confirm_tx(sendtxid) await self.generate(5) @@ -197,7 +197,7 @@ class EpicAdventuresOfChris45(CommandTestCase): channels = yield self.out(self.daemon.jsonrpc_channel_list()) self.assertEqual(len(channels), 1) self.assertEqual(channels[0]['name'], '@spam') - self.assertTrue(channels[0]['have_certificate']) + self.assertTrue(channels[0]['has_signature']) # As the new channel claim travels through the intertubes and makes its # way into the mempool and then a block and then into the claimtrie, diff --git a/tests/integration/wallet/test_transactions.py b/tests/integration/wallet/test_transactions.py index cd83324a2..1a7a4bcd6 100644 --- a/tests/integration/wallet/test_transactions.py +++ b/tests/integration/wallet/test_transactions.py @@ -46,7 +46,7 @@ class BasicTransactionTest(IntegrationTestCase): await d2f(self.account.ensure_address_gap()) - address1, address2 = await d2f(self.account.receiving.get_addresses(2, only_usable=True)) + address1, address2 = await d2f(self.account.receiving.get_addresses(limit=2, only_usable=True)) sendtxid1 = await self.blockchain.send_to_address(address1, 5) sendtxid2 = await self.blockchain.send_to_address(address2, 5) await self.blockchain.generate(1) diff --git a/tests/unit/wallet/test_ledger.py b/tests/unit/wallet/test_ledger.py index cbfc2e7be..4a587130d 100644 --- a/tests/unit/wallet/test_ledger.py +++ b/tests/unit/wallet/test_ledger.py @@ -63,7 +63,7 @@ class BasicAccountingTests(LedgerTestCase): 'insert', tx, address, hash160, '{}:{}:'.format(tx.id, 1) ) - utxos = yield self.account.get_unspent_outputs() + utxos = yield self.account.get_utxos() self.assertEqual(len(utxos), 1) tx = Transaction(is_verified=True)\ @@ -74,6 +74,6 @@ class BasicAccountingTests(LedgerTestCase): balance = yield self.account.get_balance(0, include_claims=True) self.assertEqual(balance, 0) - utxos = yield self.account.get_unspent_outputs() + utxos = yield self.account.get_utxos() self.assertEqual(len(utxos), 0) diff --git a/tests/unit/wallet/test_transaction.py b/tests/unit/wallet/test_transaction.py index 399e73317..3cab170cd 100644 --- a/tests/unit/wallet/test_transaction.py +++ b/tests/unit/wallet/test_transaction.py @@ -244,7 +244,7 @@ class TestTransactionSigning(unittest.TestCase): ) yield account.ensure_address_gap() - address1, address2 = yield account.receiving.get_addresses(2) + address1, address2 = yield account.receiving.get_addresses(limit=2) pubkey_hash1 = self.ledger.address_to_hash160(address1) pubkey_hash2 = self.ledger.address_to_hash160(address2)