From 184d3a5910ff6d4cb4c9a9a23d75910e6d8a5430 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sat, 14 Jul 2018 21:34:07 -0400 Subject: [PATCH] switched from blob to text for most columns, added a new TXRef/TXORef referencing scheme --- tests/integration/test_transactions.py | 12 +- tests/unit/test_account.py | 43 +++--- tests/unit/test_bip32.py | 2 +- tests/unit/test_ledger.py | 14 +- tests/unit/test_transaction.py | 42 +++--- tests/unit/test_wallet.py | 6 +- torba/baseaccount.py | 6 +- torba/basedatabase.py | 68 +++++----- torba/baseledger.py | 14 +- torba/basenetwork.py | 37 +----- torba/basetransaction.py | 173 +++++++++++++++---------- torba/coinselection.py | 6 +- torba/constants.py | 1 + torba/hash.py | 64 ++++++++- torba/manager.py | 18 ++- 15 files changed, 281 insertions(+), 225 deletions(-) diff --git a/tests/integration/test_transactions.py b/tests/integration/test_transactions.py index d19cb778a..6b44a90d1 100644 --- a/tests/integration/test_transactions.py +++ b/tests/integration/test_transactions.py @@ -14,9 +14,9 @@ class BasicTransactionTests(IntegrationTestCase): self.assertEqual(await self.get_balance(account2), 0) sendtxids = [] - for i in range(9): + for i in range(5): address1 = await d2f(account1.receiving.get_or_create_usable_address()) - sendtxid = await self.blockchain.send_to_address(address1.decode(), 1.1) + sendtxid = await self.blockchain.send_to_address(address1, 1.1) sendtxids.append(sendtxid) await self.on_transaction_id(sendtxid) # mempool await self.blockchain.generate(1) @@ -24,7 +24,7 @@ class BasicTransactionTests(IntegrationTestCase): self.on_transaction_id(txid) for txid in sendtxids ]) - self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 9.9) + self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 5.5) self.assertEqual(round(await self.get_balance(account2)/COIN, 1), 0) address2 = await d2f(account2.receiving.get_or_create_usable_address()) @@ -38,13 +38,9 @@ class BasicTransactionTests(IntegrationTestCase): await self.blockchain.generate(1) await self.on_transaction(tx) # confirmed - self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 7.9) + self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 3.5) self.assertEqual(round(await self.get_balance(account2)/COIN, 1), 2.0) - all_balances = await d2f(self.manager.get_balance()) - self.assertIn(self.ledger.get_id(), all_balances) - self.assertEqual(round(all_balances[self.ledger.get_id()]/COIN, 1), 9.9) - utxos = await d2f(self.account.get_unspent_outputs()) tx = await d2f(self.ledger.transaction_class.liquidate( [utxos[0]], [account1], account1 diff --git a/tests/unit/test_account.py b/tests/unit/test_account.py index f937a12d5..bc6726449 100644 --- a/tests/unit/test_account.py +++ b/tests/unit/test_account.py @@ -94,31 +94,30 @@ class TestKeyChainAccount(unittest.TestCase): def test_generate_account_from_seed(self): account = self.ledger.account_class.from_seed( self.ledger, - u"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab" - u"sent", - u"torba", receiving_gap=3, change_gap=2 + "carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab" + "sent", "torba", receiving_gap=3, change_gap=2 ) self.assertEqual( account.private_key.extended_key_string(), - b'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' - b'6yz3jMbycrLrRMpeAJxR8qDg8' + 'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' + '6yz3jMbycrLrRMpeAJxR8qDg8' ) self.assertEqual( account.public_key.extended_key_string(), - b'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' - b'iW44g14WF52fYC5J483wqQ5ZP' + 'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' + 'iW44g14WF52fYC5J483wqQ5ZP' ) address = yield account.receiving.ensure_address_gap() - self.assertEqual(address[0], b'1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP') + self.assertEqual(address[0], '1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP') - private_key = yield self.ledger.get_private_key_for_address(b'1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP') + private_key = yield self.ledger.get_private_key_for_address('1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP') self.assertEqual( private_key.extended_key_string(), - b'xprv9xNEfQ296VTRaEUDZ8oKq74xw2U6kpj486vFUB4K1wT9U25GX4UwuzFgJN1YuRrqkQ5TTwCpkYnjNpSoH' - b'SBaEigNHPkoeYbuPMRo6mRUjxg' + 'xprv9xNEfQ296VTRaEUDZ8oKq74xw2U6kpj486vFUB4K1wT9U25GX4UwuzFgJN1YuRrqkQ5TTwCpkYnjNpSoH' + 'SBaEigNHPkoeYbuPMRo6mRUjxg' ) - invalid_key = yield self.ledger.get_private_key_for_address(b'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX') + invalid_key = yield self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX') self.assertIsNone(invalid_key) self.assertEqual( @@ -247,20 +246,18 @@ class TestSingleKeyAccount(unittest.TestCase): def test_generate_account_from_seed(self): account = self.ledger.account_class.from_seed( self.ledger, - u"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab" - u"sent", - u"torba", - is_hd=False + "carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab" + "sent", "torba", is_hd=False ) self.assertEqual( account.private_key.extended_key_string(), - b'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' - b'6yz3jMbycrLrRMpeAJxR8qDg8' + 'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' + '6yz3jMbycrLrRMpeAJxR8qDg8' ) self.assertEqual( account.public_key.extended_key_string(), - b'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' - b'iW44g14WF52fYC5J483wqQ5ZP' + 'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' + 'iW44g14WF52fYC5J483wqQ5ZP' ) address = yield account.receiving.ensure_address_gap() self.assertEqual(address[0], account.public_key.address) @@ -268,11 +265,11 @@ class TestSingleKeyAccount(unittest.TestCase): private_key = yield self.ledger.get_private_key_for_address(address[0]) self.assertEqual( private_key.extended_key_string(), - b'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' - b'6yz3jMbycrLrRMpeAJxR8qDg8' + 'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' + '6yz3jMbycrLrRMpeAJxR8qDg8' ) - invalid_key = yield self.ledger.get_private_key_for_address(b'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX') + invalid_key = yield self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX') self.assertIsNone(invalid_key) self.assertEqual( diff --git a/tests/unit/test_bip32.py b/tests/unit/test_bip32.py index 90fcac263..2854da74b 100644 --- a/tests/unit/test_bip32.py +++ b/tests/unit/test_bip32.py @@ -51,7 +51,7 @@ class BIP32Tests(unittest.TestCase): self.assertEqual( ec_point.y(), 86198965946979720220333266272536217633917099472454294641561154971209433250106 ) - self.assertEqual(private_key.address(), b'1GVM5dEhThbiyCZ9gqBZBv6p9whga7MTXo' ) + self.assertEqual(private_key.address(), '1GVM5dEhThbiyCZ9gqBZBv6p9whga7MTXo' ) with self.assertRaisesRegex(ValueError, 'invalid BIP32 private key child number'): private_key.child(-1) self.assertIsInstance(private_key.child(PrivateKey.HARDENED), PrivateKey) diff --git a/tests/unit/test_ledger.py b/tests/unit/test_ledger.py index af8b51bb4..708d2fc4d 100644 --- a/tests/unit/test_ledger.py +++ b/tests/unit/test_ledger.py @@ -30,7 +30,7 @@ class MockNetwork: def get_transaction(self, tx_hash): self.get_transaction_called.append(tx_hash) - return defer.succeed(self.transaction[tx_hash.decode()]) + return defer.succeed(self.transaction[tx_hash]) class MockHeaders: @@ -70,9 +70,9 @@ class TestSynchronization(unittest.TestCase): self.ledger.headers.height = 3 self.ledger.network = MockNetwork([ - {'tx_hash': b'abcd01', 'height': 1}, - {'tx_hash': b'abcd02', 'height': 2}, - {'tx_hash': b'abcd03', 'height': 3}, + {'tx_hash': 'abcd01', 'height': 1}, + {'tx_hash': 'abcd02', 'height': 2}, + {'tx_hash': 'abcd03', 'height': 3}, ], { 'abcd01': hexlify(get_transaction(get_output(1)).raw), 'abcd02': hexlify(get_transaction(get_output(2)).raw), @@ -80,7 +80,7 @@ class TestSynchronization(unittest.TestCase): }) yield self.ledger.update_history(address) self.assertEqual(self.ledger.network.get_history_called, [address]) - self.assertEqual(self.ledger.network.get_transaction_called, [b'abcd01', b'abcd02', b'abcd03']) + self.assertEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03']) address_details = yield self.ledger.db.get_address(address) self.assertEqual(address_details['history'], 'abcd01:1:abcd02:2:abcd03:3:') @@ -91,12 +91,12 @@ class TestSynchronization(unittest.TestCase): self.assertEqual(self.ledger.network.get_history_called, [address]) self.assertEqual(self.ledger.network.get_transaction_called, []) - self.ledger.network.history.append({'tx_hash': b'abcd04', 'height': 4}) + self.ledger.network.history.append({'tx_hash': 'abcd04', 'height': 4}) self.ledger.network.transaction['abcd04'] = hexlify(get_transaction(get_output(4)).raw) self.ledger.network.get_history_called = [] self.ledger.network.get_transaction_called = [] yield self.ledger.update_history(address) self.assertEqual(self.ledger.network.get_history_called, [address]) - self.assertEqual(self.ledger.network.get_transaction_called, [b'abcd04']) + self.assertEqual(self.ledger.network.get_transaction_called, ['abcd04']) address_details = yield self.ledger.db.get_address(address) self.assertEqual(address_details['history'], 'abcd01:1:abcd02:2:abcd03:3:abcd04:4:') diff --git a/tests/unit/test_transaction.py b/tests/unit/test_transaction.py index 65b4334e8..5966cf374 100644 --- a/tests/unit/test_transaction.py +++ b/tests/unit/test_transaction.py @@ -71,11 +71,11 @@ class TestTransactionSerialization(unittest.TestCase): self.assertEqual(len(tx.outputs), 1) coinbase = tx.inputs[0] - self.assertEqual(coinbase.output_txhash, NULL_HASH) - self.assertEqual(coinbase.output_index, 0xFFFFFFFF) + self.assertTrue(coinbase.txo_ref.is_null, NULL_HASH) + self.assertEqual(coinbase.txo_ref.position, 0xFFFFFFFF) self.assertEqual(coinbase.sequence, 4294967295) - self.assertTrue(coinbase.is_coinbase) - self.assertEqual(coinbase.script, None) + self.assertIsNotNone(coinbase.coinbase) + self.assertIsNone(coinbase.script) self.assertEqual( coinbase.coinbase[8:], b'The Times 03/Jan/2009 Chancellor on brink of second bailout for banks' @@ -83,7 +83,7 @@ class TestTransactionSerialization(unittest.TestCase): out = tx.outputs[0] self.assertEqual(out.amount, 5000000000) - self.assertEqual(out.index, 0) + self.assertEqual(out.position, 0) self.assertTrue(out.script.is_pay_pubkey) self.assertFalse(out.script.is_pay_pubkey_hash) self.assertFalse(out.script.is_pay_script_hash) @@ -106,19 +106,16 @@ class TestTransactionSerialization(unittest.TestCase): self.assertEqual(len(tx.outputs), 2) coinbase = tx.inputs[0] - self.assertEqual(coinbase.output_txhash, NULL_HASH) - self.assertEqual(coinbase.output_index, 0xFFFFFFFF) + self.assertTrue(coinbase.txo_ref.is_null) + self.assertEqual(coinbase.txo_ref.position, 0xFFFFFFFF) self.assertEqual(coinbase.sequence, 4294967295) - self.assertTrue(coinbase.is_coinbase) - self.assertEqual(coinbase.script, None) - self.assertEqual( - coinbase.coinbase[9:22], - b'/BTC.COM/NYA/' - ) + self.assertIsNotNone(coinbase.coinbase) + self.assertIsNone(coinbase.script) + self.assertEqual(coinbase.coinbase[9:22], b'/BTC.COM/NYA/') out = tx.outputs[0] self.assertEqual(out.amount, 1561039505) - self.assertEqual(out.index, 0) + self.assertEqual(out.position, 0) self.assertFalse(out.script.is_pay_pubkey) self.assertFalse(out.script.is_pay_pubkey_hash) self.assertTrue(out.script.is_pay_script_hash) @@ -126,7 +123,7 @@ class TestTransactionSerialization(unittest.TestCase): out1 = tx.outputs[1] self.assertEqual(out1.amount, 0) - self.assertEqual(out1.index, 1) + self.assertEqual(out1.position, 1) self.assertEqual( hexlify(out1.script.values['data']), b'aa21a9ede6c99265a6b9e1d36c962fda0516b35709c49dc3b8176fa7e5d5f1f6197884b4' @@ -155,19 +152,20 @@ class TestTransactionSigning(unittest.TestCase): ) yield account.ensure_address_gap() - address1 = (yield account.receiving.get_addresses())[0] - address2 = (yield account.receiving.get_addresses())[0] + address1, address2 = yield account.receiving.get_addresses(2) pubkey_hash1 = self.ledger.address_to_hash160(address1) pubkey_hash2 = self.ledger.address_to_hash160(address2) - tx = ledger_class.transaction_class() \ - .add_inputs([ledger_class.transaction_class.input_class.spend(get_output(2*COIN, pubkey_hash1))]) \ - .add_outputs([ledger_class.transaction_class.output_class.pay_pubkey_hash(int(1.9*COIN), pubkey_hash2)]) \ + tx_class = ledger_class.transaction_class + + tx = tx_class() \ + .add_inputs([tx_class.input_class.spend(get_output(2*COIN, pubkey_hash1))]) \ + .add_outputs([tx_class.output_class.pay_pubkey_hash(int(1.9*COIN), pubkey_hash2)]) \ yield tx.sign([account]) self.assertEqual( hexlify(tx.inputs[0].script.values['signature']), - b'3044022064cd6b95c9e0084253c10dd56bcec2bfd816c29aad05fbea490511d79540462b02201aa9d6f73' - b'48bb0c76b28d1ad87cf4ffd51cf4de0b299af8bf0ecad70e3369ef201' + b'304402203d463519290d06891e461ea5256c56097ccdad53379b1bb4e51ec5abc6e9fd02022034ed15b9' + b'd7c678716c4aa7c0fd26c688e8f9db8075838f2839ab55d551b62c0a01' ) diff --git a/tests/unit/test_wallet.py b/tests/unit/test_wallet.py index 2adce9dde..c52119e4e 100644 --- a/tests/unit/test_wallet.py +++ b/tests/unit/test_wallet.py @@ -32,6 +32,7 @@ class TestWalletCreation(unittest.TestCase): 'name': 'Main Wallet', 'accounts': [ { + 'name': 'An Account', 'ledger': 'btc_mainnet', 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" @@ -43,10 +44,11 @@ class TestWalletCreation(unittest.TestCase): 'public_key': 'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' 'iW44g14WF52fYC5J483wqQ5ZP', + 'is_hd': True, 'receiving_gap': 10, - 'receiving_maximum_use_per_address': 2, + 'receiving_maximum_uses_per_address': 2, 'change_gap': 10, - 'change_maximum_use_per_address': 2, + 'change_maximum_uses_per_address': 2, } ] } diff --git a/torba/baseaccount.py b/torba/baseaccount.py index ec29a184a..c3ca0df37 100644 --- a/torba/baseaccount.py +++ b/torba/baseaccount.py @@ -200,7 +200,7 @@ class BaseAccount(object): def to_dict(self): private_key = self.private_key if not self.encrypted and self.private_key: - private_key = self.private_key.extended_key_string().decode() + private_key = self.private_key.extended_key_string() d = { 'ledger': self.ledger.get_id(), @@ -208,7 +208,7 @@ class BaseAccount(object): 'seed': self.seed, 'encrypted': self.encrypted, 'private_key': private_key, - 'public_key': self.public_key.extended_key_string().decode(), + 'public_key': self.public_key.extended_key_string(), 'is_hd': False } @@ -260,7 +260,7 @@ class BaseAccount(object): else: return self.private_key.child(chain).child(index) - def get_balance(self, confirmations, **constraints): + def get_balance(self, confirmations=6, **constraints): if confirmations == 0: return self.ledger.db.get_balance_for_account(self, **constraints) else: diff --git a/torba/basedatabase.py b/torba/basedatabase.py index d8c4b12c0..b274e971f 100644 --- a/torba/basedatabase.py +++ b/torba/basedatabase.py @@ -6,7 +6,7 @@ import sqlite3 from twisted.internet import defer from twisted.enterprise import adbapi -import torba.baseaccount +from torba.hash import TXRefImmutable log = logging.getLogger(__name__) @@ -131,8 +131,8 @@ class BaseDatabase(SQLiteMixin): CREATE_TXO_TABLE = """ create table if not exists txo ( - txoid text primary key, txid text references tx, + txoid text primary key, address text references pubkey_address, position integer not null, amount integer not null, @@ -158,8 +158,9 @@ class BaseDatabase(SQLiteMixin): def txo_to_row(self, tx, address, txo): return { - 'txid': sqlite3.Binary(tx.hash), - 'address': sqlite3.Binary(address), + 'txid': tx.id, + 'txoid': txo.id, + 'address': address, 'position': txo.position, 'amount': txo.amount, 'script': sqlite3.Binary(txo.script.source) @@ -170,7 +171,7 @@ class BaseDatabase(SQLiteMixin): def _steps(t): if save_tx == 'insert': t.execute(*self._insert_sql('tx', { - 'txhash': sqlite3.Binary(tx.hash), + 'txid': tx.id, 'raw': sqlite3.Binary(tx.raw), 'height': height, 'is_verified': is_verified @@ -178,16 +179,15 @@ class BaseDatabase(SQLiteMixin): elif save_tx == 'update': t.execute(*self._update_sql("tx", { 'height': height, 'is_verified': is_verified - }, 'txhash = ?', (sqlite3.Binary(tx.hash),) + }, 'txid = ?', (tx.id,) )) existing_txos = list(map(itemgetter(0), t.execute( - "SELECT position FROM txo WHERE txhash = ?", - (sqlite3.Binary(tx.hash),) + "SELECT position FROM txo WHERE txid = ?", (tx.id,) ).fetchall())) for txo in tx.outputs: - if txo.index in existing_txos: + if txo.position in existing_txos: continue if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == hash: t.execute(*self._insert_sql("txo", self.txo_to_row(tx, address, txo))) @@ -195,20 +195,17 @@ class BaseDatabase(SQLiteMixin): # TODO: implement script hash payments print('Database.save_transaction_io: pay script hash is not implemented!') - existing_txis = [txi[0] for txi in t.execute( - "SELECT txoid FROM txi WHERE txhash = ? AND address = ?", - (sqlite3.Binary(tx.hash), sqlite3.Binary(address))).fetchall()] + spent_txoids = [txi[0] for txi in t.execute( + "SELECT txoid FROM txi WHERE txid = ? AND address = ?", (tx.id, address) + ).fetchall()] for txi in tx.inputs: - txoid = t.execute( - "SELECT txoid FROM txo WHERE txhash = ? AND position = ?", - (sqlite3.Binary(txi.output_txhash), txi.output_index) - ).fetchone() - if txoid is not None and txoid[0] not in existing_txis: + txoid = txi.txo_ref.id + if txoid not in spent_txoids: t.execute(*self._insert_sql("txi", { - 'txhash': sqlite3.Binary(tx.hash), - 'address': sqlite3.Binary(address), - 'txoid': txoid[0], + 'txid': tx.id, + 'txoid': txoid, + 'address': address, })) self._set_address_history(t, address, history) @@ -225,16 +222,10 @@ class BaseDatabase(SQLiteMixin): def release_reserved_outputs(self, txoids): return self.reserve_spent_outputs(txoids, is_reserved=False) - def get_txoid_for_txo(self, txo): - return self.query_one_value( - "SELECT txoid FROM txo WHERE txhash = ? AND position = ?", - (sqlite3.Binary(txo.transaction.hash), txo.index) - ) - @defer.inlineCallbacks - def get_transaction(self, txhash): + def get_transaction(self, txid): result = yield self.db.runQuery( - "SELECT raw, height, is_verified FROM tx WHERE txhash = ?", (sqlite3.Binary(txhash),) + "SELECT raw, height, is_verified FROM tx WHERE txid = ?", (txid,) ) if result: defer.returnValue(result[0]) @@ -254,13 +245,13 @@ class BaseDatabase(SQLiteMixin): col, op = key[:-len('__lte')], '<=' extras.append('{} {} :{}'.format(col, op, key)) extra_sql = ' AND ' + ' AND '.join(extras) - values = {'account': sqlite3.Binary(account.public_key.address)} + values = {'account': account.public_key.address} values.update(constraints) result = yield self.db.runQuery( """ SELECT SUM(amount) FROM txo - JOIN tx ON tx.txhash=txo.txhash + JOIN tx ON tx.txid=txo.txid JOIN pubkey_address ON pubkey_address.address=txo.address WHERE pubkey_address.account=:account AND @@ -279,11 +270,11 @@ class BaseDatabase(SQLiteMixin): extra_sql = ' AND ' + ' AND '.join( '{} = :{}'.format(c, c) for c in constraints.keys() ) - values = {'account': sqlite3.Binary(account.public_key.address)} + values = {'account': account.public_key.address} values.update(constraints) utxos = yield self.db.runQuery( """ - SELECT amount, script, txhash, txo.position, txoid + SELECT amount, script, txid, txo.position FROM txo JOIN pubkey_address ON pubkey_address.address=txo.address WHERE account=:account AND txo.is_reserved=0 AND txoid NOT IN (SELECT txoid FROM txi) """+extra_sql, values @@ -293,9 +284,8 @@ class BaseDatabase(SQLiteMixin): output_class( values[0], output_class.script_class(values[1]), - values[2], - index=values[3], - txoid=values[4] + TXRefImmutable.from_id(values[2]), + position=values[3] ) for values in utxos ]) @@ -307,18 +297,18 @@ class BaseDatabase(SQLiteMixin): ) + ', '.join(['(?, ?, ?, ?, ?)'] * len(keys)) values = [] for position, pubkey in keys: - values.append(sqlite3.Binary(pubkey.address)) - values.append(sqlite3.Binary(account.public_key.address)) + values.append(pubkey.address) + values.append(account.public_key.address) values.append(chain) values.append(position) - values.append(sqlite3.Binary(pubkey.pubkey_bytes)) + values.append(pubkey.pubkey_bytes) return self.db.runOperation(sql, values) @staticmethod def _set_address_history(t, address, history): t.execute( "UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?", - (history, history.count(':')//2, sqlite3.Binary(address)) + (history, history.count(':')//2, address) ) def set_address_history(self, address, history): diff --git a/torba/baseledger.py b/torba/baseledger.py index fba46c48b..f62816957 100644 --- a/torba/baseledger.py +++ b/torba/baseledger.py @@ -77,7 +77,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): self.on_transaction = self._on_transaction_controller.stream self.on_transaction.listen( lambda e: log.info('({}) on_transaction: address={}, height={}, is_verified={}, tx.id={}'.format( - self.get_id(), e.address, e.height, e.is_verified, e.tx.hex_id) + self.get_id(), e.address, e.height, e.is_verified, e.tx.id) ) ) @@ -136,7 +136,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): match = yield self.db.get_address(address) if match: for account in self.accounts: - if bytes(match['account']) == account.public_key.address: + if match['account'] == account.public_key.address: defer.returnValue(account.get_private_key(match['chain'], match['position'])) @defer.inlineCallbacks @@ -178,7 +178,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): @defer.inlineCallbacks def is_valid_transaction(self, tx, height): height <= len(self.headers) or defer.returnValue(False) - merkle = yield self.network.get_merkle(tx.hex_id.decode(), height) + merkle = yield self.network.get_merkle(tx.id, height) merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash) header = self.headers[height] defer.returnValue(merkle_root == header['merkle_root']) @@ -262,7 +262,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): synced_history.append((hex_id, remote_height)) - if i < len(local_history) and local_history[i] == (hex_id.decode(), remote_height): + if i < len(local_history) and local_history[i] == (hex_id, remote_height): continue lock = self._transaction_processing_locks.setdefault(hex_id, defer.DeferredLock()) @@ -271,7 +271,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): #try: # see if we have a local copy of transaction, otherwise fetch it from server - raw, local_height, is_verified = yield self.db.get_transaction(unhexlify(hex_id)[::-1]) + raw, local_height, is_verified = yield self.db.get_transaction(hex_id) save_tx = None if raw is None: _raw = yield self.network.get_transaction(hex_id) @@ -288,7 +288,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): yield self.db.save_transaction_io( save_tx, tx, remote_height, is_verified, address, self.address_to_hash160(address), - ''.join('{}:{}:'.format(tx_id.decode(), tx_height) for tx_id, tx_height in synced_history) + ''.join('{}:{}:'.format(tx_id, tx_height) for tx_id, tx_height in synced_history) ) log.debug("{}: sync'ed tx {} for address: {}, height: {}, verified: {}".format( @@ -320,4 +320,4 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): yield self.update_history(address) def broadcast(self, tx): - return self.network.broadcast(hexlify(tx.raw)) + return self.network.broadcast(hexlify(tx.raw).decode()) diff --git a/torba/basenetwork.py b/torba/basenetwork.py index 44720a0c6..cdc74c61b 100644 --- a/torba/basenetwork.py +++ b/torba/basenetwork.py @@ -1,4 +1,3 @@ -import six import json import socket import logging @@ -13,27 +12,6 @@ from torba.stream import StreamController log = logging.getLogger(__name__) -if six.PY3: - buffer = memoryview - - -def unicode2bytes(string): - if isinstance(string, six.text_type): - return string.encode('iso-8859-1') - elif isinstance(string, list): - return [unicode2bytes(s) for s in string] - return string - - -def bytes2unicode(maybe_bytes): - if isinstance(maybe_bytes, buffer): - maybe_bytes = str(maybe_bytes) - if isinstance(maybe_bytes, bytes): - return maybe_bytes.decode() - elif isinstance(maybe_bytes, (list, tuple)): - return [bytes2unicode(b) for b in maybe_bytes] - return maybe_bytes - class StratumClientProtocol(LineOnlyReceiver): delimiter = b'\n' @@ -86,14 +64,7 @@ class StratumClientProtocol(LineOnlyReceiver): log.debug('received: {}'.format(line)) try: - # `line` comes in as a byte string but `json.loads` automatically converts everything to - # unicode. For keys it's not a big deal but for values there is an expectation - # everywhere else in wallet code that most values are byte strings. - message = json.loads( - line, object_hook=lambda obj: { - k: unicode2bytes(v) for k, v in obj.items() - } - ) + message = json.loads(line) except (ValueError, TypeError): raise ValueError("Cannot decode message '{}'".format(line.strip())) @@ -118,7 +89,7 @@ class StratumClientProtocol(LineOnlyReceiver): message = json.dumps({ 'id': message_id, 'method': method, - 'params': [bytes2unicode(arg) for arg in args] + 'params': args }) log.debug('sent: {}'.format(message)) self.sendLine(message.encode('latin-1')) @@ -160,8 +131,8 @@ class BaseNetwork: self.on_status = self._on_status_controller.stream self.subscription_controllers = { - b'blockchain.headers.subscribe': self._on_header_controller, - b'blockchain.address.subscribe': self._on_status_controller, + 'blockchain.headers.subscribe': self._on_header_controller, + 'blockchain.address.subscribe': self._on_status_controller, } @defer.inlineCallbacks diff --git a/torba/basetransaction.py b/torba/basetransaction.py index 6eaf14ac3..75c5cb45c 100644 --- a/torba/basetransaction.py +++ b/torba/basetransaction.py @@ -9,30 +9,60 @@ import torba.baseaccount import torba.baseledger from torba.basescript import BaseInputScript, BaseOutputScript from torba.coinselection import CoinSelector -from torba.constants import COIN +from torba.constants import COIN, NULL_HASH32 from torba.bcd_data_stream import BCDataStream -from torba.hash import sha256 +from torba.hash import sha256, TXRef, TXRefImmutable, TXORef from torba.util import ReadOnlyList log = logging.getLogger() -NULL_HASH = b'\x00'*32 +class TXRefMutable(TXRef): + + __slots__ = 'tx', + + def __init__(self, tx): + super(TXRefMutable, self).__init__() + self.tx = tx + + @property + def id(self): + if self._id is None: + self._id = hexlify(self.hash[::-1]).decode() + return self._id + + @property + def hash(self): + if self._hash is None: + self._hash = sha256(sha256(self.tx.raw)) + return self._hash + + def reset(self): + self._id = None + self._hash = None + + +class TXORefResolvable(TXORef): + + __slots__ = '_txo', + + def __init__(self, txo): + super(TXORefResolvable, self).__init__(txo.tx_ref, txo.position) + self._txo = txo + + @property + def txo(self): + return self._txo class InputOutput(object): - def __init__(self, txhash, index=None): - self._txhash = txhash # type: bytes - self.transaction = None # type: BaseTransaction - self.index = index # type: int + __slots__ = 'tx_ref', 'position' - @property - def txhash(self): - if self._txhash is None: - self._txhash = self.transaction.hash - return self._txhash + def __init__(self, tx_ref=None, position=None): + self.tx_ref = tx_ref # type: TXRef + self.position = position # type: int @property def size(self): @@ -52,49 +82,49 @@ class BaseInput(InputOutput): NULL_SIGNATURE = b'\x00'*72 NULL_PUBLIC_KEY = b'\x00'*33 - def __init__(self, output_or_txhash_index, script, sequence=0xFFFFFFFF, txhash=None): - super(BaseInput, self).__init__(txhash) - if isinstance(output_or_txhash_index, BaseOutput): - self.output = output_or_txhash_index # type: BaseOutput - self.output_txhash = self.output.txhash - self.output_index = self.output.index - else: - self.output = None # type: BaseOutput - self.output_txhash, self.output_index = output_or_txhash_index + __slots__ = 'txo_ref', 'sequence', 'coinbase', 'script' + + def __init__(self, txo_ref, script, sequence=0xFFFFFFFF, tx_ref=None, position=None): + # type: (TXORef, BaseInputScript, int, TXRef, int) -> None + super(BaseInput, self).__init__(tx_ref, position) + self.txo_ref = txo_ref self.sequence = sequence - self.is_coinbase = self.output_txhash == NULL_HASH - self.coinbase = script if self.is_coinbase else None - self.script = script if not self.is_coinbase else None # type: BaseInputScript + self.coinbase = script if txo_ref.is_null else None + self.script = script if not txo_ref.is_null else None # type: BaseInputScript + + @property + def is_coinbase(self): + return self.coinbase is not None @classmethod - def spend(cls, output): + def spend(cls, txo): # type: (BaseOutput) -> BaseInput """ Create an input to spend the output.""" - assert output.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.' + assert txo.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.' script = cls.script_class.redeem_pubkey_hash(cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY) - return cls(output, script) + return cls(txo.ref, script) @property def amount(self): """ Amount this input adds to the transaction. """ - if self.output is None: - raise ValueError('Cannot get input value without referenced output.') - return self.output.amount + if self.txo_ref.txo is None: + raise ValueError('Cannot resolve output to get amount.') + return self.txo_ref.txo.amount @classmethod def deserialize_from(cls, stream): - txhash = stream.read(32) - index = stream.read_uint32() + tx_ref = TXRefImmutable.from_hash(stream.read(32)) + position = stream.read_uint32() script = stream.read_string() sequence = stream.read_uint32() return cls( - (txhash, index), - cls.script_class(script) if not txhash == NULL_HASH else script, + TXORef(tx_ref, position), + cls.script_class(script) if not tx_ref.is_null else script, sequence ) def serialize_to(self, stream, alternate_script=None): - stream.write(self.output_txhash) - stream.write_uint32(self.output_index) + stream.write(self.txo_ref.tx_ref.hash) + stream.write_uint32(self.txo_ref.position) if alternate_script is not None: stream.write_string(alternate_script) else: @@ -107,7 +137,7 @@ class BaseInput(InputOutput): class BaseOutputEffectiveAmountEstimator(object): - __slots__ = 'coin', 'txi', 'txo', 'fee', 'effective_amount' + __slots__ = 'txo', 'txi', 'fee', 'effective_amount' def __init__(self, ledger, txo): # type: (torba.baseledger.BaseLedger, BaseOutput) -> None self.txo = txo @@ -124,11 +154,21 @@ class BaseOutput(InputOutput): script_class = BaseOutputScript estimator_class = BaseOutputEffectiveAmountEstimator - def __init__(self, amount, script, txhash=None, index=None, txoid=None): - super(BaseOutput, self).__init__(txhash, index) - self.amount = amount # type: int - self.script = script # type: BaseOutputScript - self.txoid = txoid + __slots__ = 'amount', 'script' + + def __init__(self, amount, script, tx_ref=None, position=None): + # type: (int, BaseOutputScript, TXRef, int) -> None + super(BaseOutput, self).__init__(tx_ref, position) + self.amount = amount + self.script = script + + @property + def ref(self): + return TXORefResolvable(self) + + @property + def id(self): + return self.ref.id def get_estimator(self, ledger): return self.estimator_class(ledger, self) @@ -156,8 +196,7 @@ class BaseTransaction: def __init__(self, raw=None, version=1, locktime=0): self._raw = raw - self._hash = None - self._id = None + self.ref = TXRefMutable(self) self.version = version # type: int self.locktime = locktime # type: int self._inputs = [] # type: List[BaseInput] @@ -165,21 +204,13 @@ class BaseTransaction: if raw is not None: self._deserialize() - @property - def hex_id(self): - return hexlify(self.id) - @property def id(self): - if self._id is None: - self._id = self.hash[::-1] - return self._id + return self.ref.id @property def hash(self): - if self._hash is None: - self._hash = sha256(sha256(self.raw)) - return self._hash + return self.ref.hash @property def raw(self): @@ -188,9 +219,8 @@ class BaseTransaction: return self._raw def _reset(self): - self._id = None - self._hash = None self._raw = None + self.ref.reset() @property def inputs(self): # type: () -> ReadOnlyList[BaseInput] @@ -201,35 +231,36 @@ class BaseTransaction: return ReadOnlyList(self._outputs) def _add(self, new_ios, existing_ios): + # type: (List[InputOutput], List[InputOutput]) -> BaseTransaction for txio in new_ios: - txio.transaction = self - txio.index = len(existing_ios) + txio.tx_ref = self.ref + txio.position = len(existing_ios) existing_ios.append(txio) self._reset() return self - def add_inputs(self, inputs): + def add_inputs(self, inputs): # type: (List[BaseInput]) -> BaseTransaction return self._add(inputs, self._inputs) - def add_outputs(self, outputs): + def add_outputs(self, outputs): # type: (List[BaseOutput]) -> BaseTransaction return self._add(outputs, self._outputs) @property - def fee(self): + def fee(self): # type: () -> int """ Fee that will actually be paid.""" return self.input_sum - self.output_sum @property - def size(self): + def size(self): # type: () -> int """ Size in bytes of the entire transaction. """ return len(self.raw) @property - def base_size(self): + def base_size(self): # type: () -> int """ Size in bytes of transaction meta data and all outputs; without inputs. """ return len(self._serialize(with_inputs=False)) - def _serialize(self, with_inputs=True): + def _serialize(self, with_inputs=True): # type: (bool) -> bytes stream = BCDataStream() stream.write_uint32(self.version) if with_inputs: @@ -242,13 +273,13 @@ class BaseTransaction: stream.write_uint32(self.locktime) return stream.get_bytes() - def _serialize_for_signature(self, signing_input): + def _serialize_for_signature(self, signing_input): # type: (int) -> bytes stream = BCDataStream() stream.write_uint32(self.version) stream.write_compact_size(len(self._inputs)) for i, txin in enumerate(self._inputs): if signing_input == i: - txin.serialize_to(stream, txin.output.script.source) + txin.serialize_to(stream, txin.txo_ref.txo.script.source) else: txin.serialize_to(stream, b'') stream.write_compact_size(len(self._outputs)) @@ -300,7 +331,7 @@ class BaseTransaction: selector = CoinSelector( txos, amount, ledger.get_input_output_fee( - cls.output_class.pay_pubkey_hash(COIN, NULL_HASH) + cls.output_class.pay_pubkey_hash(COIN, NULL_HASH32) ) ) @@ -308,7 +339,7 @@ class BaseTransaction: if not spendables: raise ValueError('Not enough funds to cover this transaction.') - reserved_outputs = [s.txo.txoid for s in spendables] + reserved_outputs = [s.txo.id for s in spendables] if reserve_outputs: yield ledger.db.reserve_spent_outputs(reserved_outputs) @@ -340,14 +371,14 @@ class BaseTransaction: ]) ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account) - reserved_outputs = [utxo.txoid for utxo in assets] + reserved_outputs = [utxo.id for utxo in assets] if reserve_outputs: yield ledger.db.reserve_spent_outputs(reserved_outputs) try: cost_of_change = ( ledger.get_transaction_base_fee(tx) + - ledger.get_input_output_fee(cls.output_class.pay_pubkey_hash(COIN, NULL_HASH)) + ledger.get_input_output_fee(cls.output_class.pay_pubkey_hash(COIN, NULL_HASH32)) ) liquidated_total = sum(utxo.amount for utxo in assets) if liquidated_total > cost_of_change: @@ -372,7 +403,7 @@ class BaseTransaction: def sign(self, funding_accounts): # type: (Iterable[torba.baseaccount.BaseAccount]) -> BaseTransaction ledger = self.ensure_all_have_same_ledger(funding_accounts) for i, txi in enumerate(self._inputs): - txo_script = txi.output.script + txo_script = txi.txo_ref.txo.script if txo_script.is_pay_pubkey_hash: address = ledger.hash160_to_address(txo_script.values['pubkey_hash']) private_key = yield ledger.get_private_key_for_address(address) diff --git a/torba/coinselection.py b/torba/coinselection.py index 398781676..0c1adede0 100644 --- a/torba/coinselection.py +++ b/torba/coinselection.py @@ -21,14 +21,14 @@ class CoinSelector: if six.PY3 and seed is not None: self.random.seed(seed, version=1) - def select(self): + def select(self): # type: () -> List[torba.basetransaction.BaseOutputAmountEstimator] if not self.txos: return if self.target > self.available: return return self.branch_and_bound() or self.single_random_draw() - def branch_and_bound(self): + def branch_and_bound(self): # type: () -> List[torba.basetransaction.BaseOutputAmountEstimator] # see bitcoin implementation for more info: # https://github.com/bitcoin/bitcoin/blob/master/src/wallet/coinselection.cpp @@ -84,7 +84,7 @@ class CoinSelector: self.txos[i] for i, include in enumerate(best_selection) if include ] - def single_random_draw(self): + def single_random_draw(self): # type: () -> List[torba.basetransaction.BaseOutputAmountEstimator] self.random.shuffle(self.txos, self.random.random) selection = [] amount = 0 diff --git a/torba/constants.py b/torba/constants.py index 9fab12b5f..c1497ac19 100644 --- a/torba/constants.py +++ b/torba/constants.py @@ -1,3 +1,4 @@ +NULL_HASH32 = b'\x00'*32 CENT = 1000000 COIN = 100*CENT diff --git a/torba/hash.py b/torba/hash.py index 3d7092e58..d8ab42bb6 100644 --- a/torba/hash.py +++ b/torba/hash.py @@ -20,6 +20,7 @@ from cryptography.hazmat.primitives.padding import PKCS7 from cryptography.hazmat.backends import default_backend from torba.util import bytes_to_int, int_to_bytes +from torba.constants import NULL_HASH32 _sha256 = hashlib.sha256 _sha512 = hashlib.sha512 @@ -27,6 +28,67 @@ _new_hash = hashlib.new _new_hmac = hmac.new +class TXRef(object): + + __slots__ = '_id', '_hash' + + def __init__(self): + self._id = None + self._hash = None + + @property + def id(self): + return self._id + + @property + def hash(self): + return self._hash + + @property + def is_null(self): + return self.hash == NULL_HASH32 + + +class TXRefImmutable(TXRef): + + __slots__ = () + + @classmethod + def from_hash(cls, tx_hash): # type: (bytes) -> TXRefImmutable + ref = cls() + ref._hash = tx_hash + ref._id = hexlify(tx_hash[::-1]).decode() + return ref + + @classmethod + def from_id(cls, tx_id): # type: (str) -> TXRefImmutable + ref = cls() + ref._id = tx_id + ref._hash = unhexlify(tx_id)[::-1] + return ref + + +class TXORef(object): + + __slots__ = 'tx_ref', 'position' + + def __init__(self, tx_ref, position): # type: (TXRef, int) -> None + self.tx_ref = tx_ref + self.position = position + + @property + def id(self): + return '{}:{}'.format(self.tx_ref.id, self.position) + + @property + def is_null(self): + return self.tx_ref.is_null + + @property + def txo(self): + return None + + def sha256(x): """ Simple wrapper of hashlib sha256. """ return _sha256(x).digest() @@ -164,7 +226,7 @@ class Base58(object): break txt += u'1' - return txt[::-1].encode() + return txt[::-1] @classmethod def decode_check(cls, txt, hash_fn=double_sha256): diff --git a/torba/manager.py b/torba/manager.py index f535be24e..ac803e419 100644 --- a/torba/manager.py +++ b/torba/manager.py @@ -1,8 +1,10 @@ +from decimal import Decimal from typing import List, Dict, Type from twisted.internet import defer from torba.baseledger import BaseLedger, LedgerRegistry from torba.wallet import Wallet, WalletStorage +from torba.constants import COIN class WalletManager(object): @@ -39,12 +41,18 @@ class WalletManager(object): return wallet @defer.inlineCallbacks - def get_balance(self): + def get_balances(self, confirmations=6): balances = {} - for ledger in self.ledgers.values(): - for account in ledger.accounts: - balances.setdefault(ledger.get_id(), 0) - balances[ledger.get_id()] += yield account.get_balance() + for i, ledger in enumerate(self.ledgers.values()): + ledger_balances = balances[ledger.get_id()] = [] + for j, account in enumerate(ledger.accounts): + satoshis = yield account.get_balance(confirmations) + ledger_balances.append({ + 'account': account.name, + 'coins': round(Decimal(satoshis) / COIN, 2), + 'satoshis': satoshis, + 'is_default_account': i == 0 and j == 0 + }) defer.returnValue(balances) @property