From e9fbc875e08bc379c8b71cc9e0b21e2039583ee0 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Wed, 13 Jun 2018 20:57:57 -0400 Subject: [PATCH] passing tests --- .travis.yml | 6 +- tests/__init__.py | 0 tests/integration/test_transactions.py | 17 ++--- tests/unit/test_account.py | 4 +- tests/unit/test_coinselection.py | 22 +++--- tests/unit/test_ledger.py | 8 ++- tests/unit/test_transaction.py | 94 +++++++++++++------------- tests/unit/test_wallet.py | 72 +++++--------------- torba/baseaccount.py | 6 +- torba/basedatabase.py | 62 +++++++++++------ torba/baseledger.py | 26 +++---- torba/basenetwork.py | 2 +- torba/basetransaction.py | 79 +++++++++++----------- torba/coin/bitcoincash.py | 71 ++++++------------- torba/coin/bitcoinsegwit.py | 1 + torba/hash.py | 3 + torba/wallet.py | 39 +++-------- tox.ini | 12 ++-- 18 files changed, 231 insertions(+), 293 deletions(-) delete mode 100644 tests/__init__.py diff --git a/.travis.yml b/.travis.yml index e37918195..bdd37d199 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,11 @@ python: - "2.7" - "3.6" -install: pip install tox-travis coverage +install: + - pip install tox-travis coverage + - pushd .. && git clone https://github.com/lbryio/electrumx.git --branch packages && popd + - pushd .. && git clone https://github.com/lbryio/orchstr8.git && popd + script: tox after_success: diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/integration/test_transactions.py b/tests/integration/test_transactions.py index 9cf9327aa..4de043124 100644 --- a/tests/integration/test_transactions.py +++ b/tests/integration/test_transactions.py @@ -17,22 +17,23 @@ class BasicTransactionTests(IntegrationTestCase): address = await account1.receiving.get_or_create_usable_address().asFuture(asyncio.get_event_loop()) sendtxid = await self.blockchain.send_to_address(address.decode(), 5.5) - await self.on_transaction(sendtxid) + await self.on_transaction(sendtxid) #mempool await self.blockchain.generate(1) - await asyncio.sleep(5) + await self.on_transaction(sendtxid) #confirmed self.assertEqual(await self.get_balance(account1), int(5.5*COIN)) self.assertEqual(await self.get_balance(account2), 0) address = await account2.receiving.get_or_create_usable_address().asFuture(asyncio.get_event_loop()) tx = await self.ledger.transaction_class.pay( - [self.ledger.transaction_class.output_class.pay_pubkey_hash(2, self.ledger.address_to_hash160(address))], + [self.ledger.transaction_class.output_class.pay_pubkey_hash(2*COIN, self.ledger.address_to_hash160(address))], [account1], account1 ).asFuture(asyncio.get_event_loop()) + await self.blockchain.decode_raw_transaction(tx) await self.broadcast(tx) - await self.on_transaction(tx.id.decode()) - await self.lbrycrd.generate(1) - - self.assertEqual(await self.get_balance(account1), int(3.5*COIN)) - self.assertEqual(await self.get_balance(account2), int(2.0*COIN)) + await self.on_transaction(tx.hex_id.decode()) #mempool + await self.blockchain.generate(1) + await self.on_transaction(tx.hex_id.decode()) #confirmed + self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 3.5) + self.assertEqual(round(await self.get_balance(account2)/COIN, 1), 2.0) diff --git a/tests/unit/test_account.py b/tests/unit/test_account.py index 393def1d4..c62e9fcaf 100644 --- a/tests/unit/test_account.py +++ b/tests/unit/test_account.py @@ -41,13 +41,13 @@ class TestKeyChain(unittest.TestCase): # case #2: only one new addressed needed keys = yield account.receiving.get_addresses(None, True) - yield self.ledger.db.set_address_history(keys[19]['address'], 'a:1:') + yield self.ledger.db.set_address_history(keys[19]['address'], b'a:1:') new_keys = yield account.receiving.ensure_address_gap() self.assertEqual(len(new_keys), 1) # case #3: 20 addresses needed keys = yield account.receiving.get_addresses(None, True) - yield self.ledger.db.set_address_history(keys[0]['address'], 'a:1:') + yield self.ledger.db.set_address_history(keys[0]['address'], b'a:1:') new_keys = yield account.receiving.ensure_address_gap() self.assertEqual(len(new_keys), 20) diff --git a/tests/unit/test_coinselection.py b/tests/unit/test_coinselection.py index 2e07ee73a..c6f8582db 100644 --- a/tests/unit/test_coinselection.py +++ b/tests/unit/test_coinselection.py @@ -1,11 +1,11 @@ -import unittest +from twisted.trial import unittest +from types import GeneratorType from torba.coin.bitcoinsegwit import MainNetLedger from torba.coinselection import CoinSelector, MAXIMUM_TRIES from torba.constants import CENT -from torba.manager import WalletManager -from .test_transaction import Output, get_output as utxo +from .test_transaction import get_output as utxo NULL_HASH = b'\x00'*32 @@ -13,7 +13,7 @@ NULL_HASH = b'\x00'*32 def search(*args, **kwargs): selection = CoinSelector(*args, **kwargs).branch_and_bound() - return [o.output.amount for o in selection] if selection else selection + return [o.txo.amount for o in selection] if selection else selection class BaseSelectionTestCase(unittest.TestCase): @@ -23,7 +23,7 @@ class BaseSelectionTestCase(unittest.TestCase): return self.ledger.db.start() def estimates(self, *args): - txos = args if isinstance(args[0], Output) else args[0] + txos = args[0] if isinstance(args[0], (GeneratorType, list)) else args return [txo.get_estimator(self.ledger) for txo in txos] @@ -44,7 +44,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase): self.assertEqual(selector.tries, 201) def test_exact_match(self): - fee = utxo(CENT).get_estimator(self.coin).fee + fee = utxo(CENT).get_estimator(self.ledger).fee utxo_pool = self.estimates( utxo(CENT + fee), utxo(CENT), @@ -52,7 +52,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase): ) selector = CoinSelector(utxo_pool, CENT, 0) match = selector.select() - self.assertEqual([CENT + fee], [c.output.amount for c in match]) + self.assertEqual([CENT + fee], [c.txo.amount for c in match]) self.assertTrue(selector.exact_match) def test_random_draw(self): @@ -63,7 +63,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase): ) selector = CoinSelector(utxo_pool, CENT, 0, '\x00') match = selector.select() - self.assertEqual([2 * CENT], [c.output.amount for c in match]) + self.assertEqual([2 * CENT], [c.txo.amount for c in match]) self.assertFalse(selector.exact_match) @@ -78,10 +78,6 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase): # Branch and Bound coin selection white paper: # https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf - def setUp(self): - ledger = WalletManager().get_or_create_ledger(BTC.get_id()) - self.coin = BTC(ledger, 0) - def make_hard_case(self, utxos): target = 0 utxo_pool = [] @@ -93,6 +89,8 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase): return self.estimates(utxo_pool), target def test_branch_and_bound_coin_selection(self): + self.ledger.fee_per_byte = 0 + utxo_pool = self.estimates( utxo(1 * CENT), utxo(2 * CENT), diff --git a/tests/unit/test_ledger.py b/tests/unit/test_ledger.py index eb281888a..0a354648d 100644 --- a/tests/unit/test_ledger.py +++ b/tests/unit/test_ledger.py @@ -1,3 +1,4 @@ +import six from binascii import hexlify from twisted.trial import unittest from twisted.internet import defer @@ -6,6 +7,9 @@ from torba.coin.bitcoinsegwit import MainNetLedger from .test_transaction import get_transaction +if six.PY3: + buffer = memoryview + class MockNetwork: @@ -53,7 +57,7 @@ class TestSynchronization(unittest.TestCase): self.assertEqual(self.ledger.network.get_transaction_called, [b'abc', b'def', b'ghi']) address_details = yield self.ledger.db.get_address(address) - self.assertEqual(address_details['history'], b'abc:1:def:2:ghi:3:') + self.assertEqual(address_details['history'], buffer(b'abc:1:def:2:ghi:3:')) self.ledger.network.get_history_called = [] self.ledger.network.get_transaction_called = [] @@ -69,4 +73,4 @@ class TestSynchronization(unittest.TestCase): self.assertEqual(self.ledger.network.get_history_called, [address]) self.assertEqual(self.ledger.network.get_transaction_called, [b'jkl']) address_details = yield self.ledger.db.get_address(address) - self.assertEqual(address_details['history'], b'abc:1:def:2:ghi:3:jkl:4:') + self.assertEqual(address_details['history'], buffer(b'abc:1:def:2:ghi:3:jkl:4:')) diff --git a/tests/unit/test_transaction.py b/tests/unit/test_transaction.py index 319658714..afb2b0da7 100644 --- a/tests/unit/test_transaction.py +++ b/tests/unit/test_transaction.py @@ -1,8 +1,8 @@ from binascii import hexlify, unhexlify from twisted.trial import unittest +from twisted.internet import defer -from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput -from torba.coin.bitcoinsegwit import MainNetLedger +from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class from torba.constants import CENT, COIN from torba.manager import WalletManager from torba.wallet import Wallet @@ -14,30 +14,26 @@ FEE_PER_CHAR = 200000 def get_output(amount=CENT, pubkey_hash=NULL_HASH): - return BaseTransaction() \ - .add_outputs([BaseTransaction.output_class.pay_pubkey_hash(amount, pubkey_hash)]) \ + return ledger_class.transaction_class() \ + .add_outputs([ledger_class.transaction_class.output_class.pay_pubkey_hash(amount, pubkey_hash)]) \ .outputs[0] def get_input(): - return BaseInput.spend(get_output()) + return ledger_class.transaction_class.input_class.spend(get_output()) def get_transaction(txo=None): - return BaseTransaction() \ + return ledger_class.transaction_class() \ .add_inputs([get_input()]) \ - .add_outputs([txo or BaseOutput.pay_pubkey_hash(CENT, NULL_HASH)]) - - -def get_wallet_and_ledger(): - ledger = WalletManager().get_or_create_ledger(MainNetLedger.get_id()) - return Wallet('Main', [ledger], [ledger.account_class.generate(ledger, u'torba')]), ledger + .add_outputs([txo or ledger_class.transaction_class.output_class.pay_pubkey_hash(CENT, NULL_HASH)]) class TestSizeAndFeeEstimation(unittest.TestCase): def setUp(self): - self.wallet, self.ledger = get_wallet_and_ledger() + self.ledger = ledger_class(db=':memory:') + return self.ledger.db.start() def io_fee(self, io): return self.ledger.get_input_output_fee(io) @@ -70,20 +66,20 @@ class TestTransactionSerialization(unittest.TestCase): '000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4c' 'ef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000' ) - tx = BaseTransaction(raw) + tx = ledger_class.transaction_class(raw) self.assertEqual(tx.version, 1) self.assertEqual(tx.locktime, 0) self.assertEqual(len(tx.inputs), 1) self.assertEqual(len(tx.outputs), 1) - ledgerbase = tx.inputs[0] - self.assertEqual(ledgerbase.output_txid, NULL_HASH) - self.assertEqual(ledgerbase.output_index, 0xFFFFFFFF) - self.assertEqual(ledgerbase.sequence, 4294967295) - self.assertTrue(ledgerbase.is_ledgerbase) - self.assertEqual(ledgerbase.script, None) + coinbase = tx.inputs[0] + self.assertEqual(coinbase.output_txhash, NULL_HASH) + self.assertEqual(coinbase.output_index, 0xFFFFFFFF) + self.assertEqual(coinbase.sequence, 4294967295) + self.assertTrue(coinbase.is_coinbase) + self.assertEqual(coinbase.script, None) self.assertEqual( - ledgerbase.ledgerbase[8:], + coinbase.coinbase[8:], b'The Times 03/Jan/2009 Chancellor on brink of second bailout for banks' ) @@ -97,7 +93,7 @@ class TestTransactionSerialization(unittest.TestCase): tx._reset() self.assertEqual(tx.raw, raw) - def test_ledgerbase_transaction(self): + def test_coinbase_transaction(self): raw = unhexlify( '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4e03' '1f5a070473319e592f4254432e434f4d2f4e59412ffabe6d6dcceb2a9d0444c51cabc4ee97a1a000036ca0' @@ -105,20 +101,20 @@ class TestTransactionSerialization(unittest.TestCase): '0000000017a914e083685a1097ce1ea9e91987ab9e94eae33d8a13870000000000000000266a24aa21a9ed' 'e6c99265a6b9e1d36c962fda0516b35709c49dc3b8176fa7e5d5f1f6197884b400000000' ) - tx = BaseTransaction(raw) + tx = ledger_class.transaction_class(raw) self.assertEqual(tx.version, 1) self.assertEqual(tx.locktime, 0) self.assertEqual(len(tx.inputs), 1) self.assertEqual(len(tx.outputs), 2) - ledgerbase = tx.inputs[0] - self.assertEqual(ledgerbase.output_txid, NULL_HASH) - self.assertEqual(ledgerbase.output_index, 0xFFFFFFFF) - self.assertEqual(ledgerbase.sequence, 4294967295) - self.assertTrue(ledgerbase.is_ledgerbase) - self.assertEqual(ledgerbase.script, None) + coinbase = tx.inputs[0] + self.assertEqual(coinbase.output_txhash, NULL_HASH) + self.assertEqual(coinbase.output_index, 0xFFFFFFFF) + self.assertEqual(coinbase.sequence, 4294967295) + self.assertTrue(coinbase.is_coinbase) + self.assertEqual(coinbase.script, None) self.assertEqual( - ledgerbase.ledgerbase[9:22], + coinbase.coinbase[9:22], b'/BTC.COM/NYA/' ) @@ -148,26 +144,32 @@ class TestTransactionSerialization(unittest.TestCase): class TestTransactionSigning(unittest.TestCase): + def setUp(self): + self.ledger = ledger_class(db=':memory:') + return self.ledger.db.start() + + @defer.inlineCallbacks def test_sign(self): - ledger = WalletManager().get_or_create_ledger(BTC.get_id()) - ledger = BTC(ledger) - wallet = Wallet('Main', [ledger], [Account.from_seed( - ledger, u'carbon smart garage balance margin twelve chest sword toast envelope bottom stom' - u'ach absent', u'torba' - )]) - account = wallet.default_account + account = self.ledger.account_class.from_seed( + self.ledger, + u"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab" + u"sent", u"torba" + ) - address1 = account.receiving_keys.generate_next_address() - address2 = account.receiving_keys.generate_next_address() - pubkey_hash1 = account.ledger.address_to_hash160(address1) - pubkey_hash2 = account.ledger.address_to_hash160(address2) + yield account.ensure_address_gap() + address1 = (yield account.receiving.get_addresses())[0] + address2 = (yield account.receiving.get_addresses())[0] + pubkey_hash1 = self.ledger.address_to_hash160(address1) + pubkey_hash2 = self.ledger.address_to_hash160(address2) - tx = Transaction() \ - .add_inputs([Input.spend(get_output(2*COIN, pubkey_hash1))]) \ - .add_outputs([Output.pay_pubkey_hash(int(1.9*COIN), pubkey_hash2)]) \ - .sign(account) + tx = ledger_class.transaction_class() \ + .add_inputs([ledger_class.transaction_class.input_class.spend(get_output(2*COIN, pubkey_hash1))]) \ + .add_outputs([ledger_class.transaction_class.output_class.pay_pubkey_hash(int(1.9*COIN), pubkey_hash2)]) \ + + yield tx.sign([account]) self.assertEqual( hexlify(tx.inputs[0].script.values['signature']), - b'304402203d463519290d06891e461ea5256c56097ccdad53379b1bb4e51ec5abc6e9fd02022034ed15b9d7c678716c4aa7c0fd26c688e8f9db8075838f2839ab55d551b62c0a01' + b'3044022064cd6b95c9e0084253c10dd56bcec2bfd816c29aad05fbea490511d79540462b02201aa9d6f73' + b'48bb0c76b28d1ad87cf4ffd51cf4de0b299af8bf0ecad70e3369ef201' ) diff --git a/tests/unit/test_wallet.py b/tests/unit/test_wallet.py index 63314303e..86521fdd5 100644 --- a/tests/unit/test_wallet.py +++ b/tests/unit/test_wallet.py @@ -1,75 +1,50 @@ from twisted.trial import unittest -from torba.coin.bitcoinsegwit import BTC +from torba.coin.bitcoinsegwit import MainNetLedger as BTCLedger +from torba.coin.bitcoincash import MainNetLedger as BCHLedger from torba.manager import WalletManager -from torba.wallet import Account, Wallet, WalletStorage - -from .ftc import FTC +from torba.wallet import Wallet, WalletStorage class TestWalletCreation(unittest.TestCase): def setUp(self): self.manager = WalletManager() - self.btc_ledger = self.manager.get_or_create_ledger(BTC.get_id()) - self.ftc_ledger = self.manager.get_or_create_ledger(FTC.get_id()) + config = {'wallet_path': '/tmp/wallet'} + self.btc_ledger = self.manager.get_or_create_ledger(BTCLedger.get_id(), config) + self.bch_ledger = self.manager.get_or_create_ledger(BCHLedger.get_id(), config) def test_create_wallet_and_accounts(self): wallet = Wallet() self.assertEqual(wallet.name, 'Wallet') - self.assertEqual(wallet.coins, []) self.assertEqual(wallet.accounts, []) account1 = wallet.generate_account(self.btc_ledger) - account2 = wallet.generate_account(self.btc_ledger) - account3 = wallet.generate_account(self.ftc_ledger) + wallet.generate_account(self.btc_ledger) + wallet.generate_account(self.bch_ledger) self.assertEqual(wallet.default_account, account1) - self.assertEqual(len(wallet.coins), 2) self.assertEqual(len(wallet.accounts), 3) - self.assertIsInstance(wallet.coins[0], BTC) - self.assertIsInstance(wallet.coins[1], FTC) - - self.assertEqual(len(account1.receiving_keys.addresses), 0) - self.assertEqual(len(account1.change_keys.addresses), 0) - self.assertEqual(len(account2.receiving_keys.addresses), 0) - self.assertEqual(len(account2.change_keys.addresses), 0) - self.assertEqual(len(account3.receiving_keys.addresses), 0) - self.assertEqual(len(account3.change_keys.addresses), 0) - account1.ensure_enough_addresses() - account2.ensure_enough_addresses() - account3.ensure_enough_addresses() - self.assertEqual(len(account1.receiving_keys.addresses), 20) - self.assertEqual(len(account1.change_keys.addresses), 6) - self.assertEqual(len(account2.receiving_keys.addresses), 20) - self.assertEqual(len(account2.change_keys.addresses), 6) - self.assertEqual(len(account3.receiving_keys.addresses), 20) - self.assertEqual(len(account3.change_keys.addresses), 6) def test_load_and_save_wallet(self): wallet_dict = { 'name': 'Main Wallet', 'accounts': [ { - 'coin': 'btc_mainnet', + 'ledger': 'btc_mainnet', 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': - 'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' - '6yz3jMbycrLrRMpeAJxR8qDg8', + b'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' + b'6yz3jMbycrLrRMpeAJxR8qDg8', 'public_key': - 'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' - 'iW44g14WF52fYC5J483wqQ5ZP', + b'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' + b'iW44g14WF52fYC5J483wqQ5ZP', 'receiving_gap': 10, - 'receiving_keys': [ - '0222345947a59dca4a3363ffa81ac87dd907d2b2feff57383eaeddbab266ca5f2d', - '03fdc9826d5d00a484188cba8eb7dba5877c0323acb77905b7bcbbab35d94be9f6' - ], + 'receiving_maximum_use_per_address': 2, 'change_gap': 10, - 'change_keys': [ - '038836be4147836ed6b4df6a89e0d9f1b1c11cec529b7ff5407de57f2e5b032c83' - ] + 'change_maximum_use_per_address': 2, } ] } @@ -77,21 +52,8 @@ class TestWalletCreation(unittest.TestCase): storage = WalletStorage(default=wallet_dict) wallet = Wallet.from_storage(storage, self.manager) self.assertEqual(wallet.name, 'Main Wallet') - self.assertEqual(len(wallet.coins), 1) - self.assertIsInstance(wallet.coins[0], BTC) self.assertEqual(len(wallet.accounts), 1) account = wallet.default_account - self.assertIsInstance(account, Account) - - self.assertEqual(len(account.receiving_keys.addresses), 2) - self.assertEqual( - account.receiving_keys.addresses[0], - b'1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP' - ) - self.assertEqual(len(account.change_keys.addresses), 1) - self.assertEqual( - account.change_keys.addresses[0], - b'1PUbu1D1f3c244JPRSJKBCxRqui5NT6geR' - ) - wallet_dict['coins'] = {'btc_mainnet': {'fee_per_byte': 50}} + self.assertIsInstance(account, BTCLedger.account_class) + self.maxDiff = None self.assertDictEqual(wallet_dict, wallet.to_dict()) diff --git a/torba/baseaccount.py b/torba/baseaccount.py index 194f287ba..f753732df 100644 --- a/torba/baseaccount.py +++ b/torba/baseaccount.py @@ -59,9 +59,9 @@ class KeyChain: def get_or_create_usable_address(self): addresses = yield self.get_usable_addresses(1) if addresses: - return addresses[0] + defer.returnValue(addresses[0]) addresses = yield self.ensure_address_gap() - return addresses[0] + defer.returnValue(addresses[0]) class BaseAccount: @@ -83,7 +83,7 @@ class BaseAccount: KeyChain(self, public_key, 0, receiving_gap, receiving_maximum_use_per_address), KeyChain(self, public_key, 1, change_gap, change_maximum_use_per_address) ) - ledger.account_created(self) + ledger.add_account(self) @classmethod def generate(cls, ledger, password): # type: (torba.baseledger.BaseLedger, str) -> BaseAccount diff --git a/torba/basedatabase.py b/torba/basedatabase.py index 5f7d4f2a6..378fbdea3 100644 --- a/torba/basedatabase.py +++ b/torba/basedatabase.py @@ -1,5 +1,6 @@ import logging from typing import List, Union +from operator import itemgetter import sqlite3 from twisted.internet import defer @@ -97,11 +98,10 @@ class BaseDatabase(SQLiteMixin): CREATE_TX_TABLE = """ create table if not exists tx ( - txid blob primary key, + txhash blob primary key, raw blob not null, height integer not null, - is_confirmed boolean not null, - is_verified boolean not null + is_verified boolean not null default false ); """ @@ -113,14 +113,14 @@ class BaseDatabase(SQLiteMixin): position integer not null, pubkey blob not null, history text, - used_times integer default 0 + used_times integer not null default 0 ); """ CREATE_TXO_TABLE = """ create table if not exists txo ( txoid integer primary key, - txid blob references tx, + txhash blob references tx, address blob references pubkey_address, position integer not null, amount integer not null, @@ -130,7 +130,7 @@ class BaseDatabase(SQLiteMixin): CREATE_TXI_TABLE = """ create table if not exists txi ( - txid blob references tx, + txhash blob references tx, address blob references pubkey_address, txoid integer references txo ); @@ -143,20 +143,33 @@ class BaseDatabase(SQLiteMixin): CREATE_TXI_TABLE ) - def add_transaction(self, address, hash, tx, height, is_confirmed, is_verified): + def add_transaction(self, address, hash, tx, height, is_verified): def _steps(t): - if not t.execute("SELECT 1 FROM tx WHERE txid=?", (sqlite3.Binary(tx.id),)).fetchone(): + current_height = t.execute("SELECT height FROM tx WHERE txhash=?", (sqlite3.Binary(tx.hash),)).fetchone() + if current_height is None: t.execute(*self._insert_sql('tx', { - 'txid': sqlite3.Binary(tx.id), + 'txhash': sqlite3.Binary(tx.hash), 'raw': sqlite3.Binary(tx.raw), 'height': height, - 'is_confirmed': is_confirmed, 'is_verified': is_verified })) + elif current_height[0] != height: + t.execute("UPDATE tx SET height = :height WHERE txhash = :txhash", { + 'txhash': sqlite3.Binary(tx.hash), + 'height': height, + }) + + existing_txos = list(map(itemgetter(0), t.execute( + "SELECT position FROM txo WHERE txhash = ?", + (sqlite3.Binary(tx.hash),) + ).fetchall())) + for txo in tx.outputs: + if txo.index in existing_txos: + continue if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == hash: t.execute(*self._insert_sql("txo", { - 'txid': sqlite3.Binary(tx.id), + 'txhash': sqlite3.Binary(tx.hash), 'address': sqlite3.Binary(address), 'position': txo.index, 'amount': txo.amount, @@ -166,24 +179,33 @@ class BaseDatabase(SQLiteMixin): # TODO: implement script hash payments print('Database.add_transaction pay script hash is not implemented!') + existing_txis = [txi[0] for txi in t.execute( + "SELECT txoid FROM txi WHERE txhash = ? AND address = ?", + (sqlite3.Binary(tx.hash), sqlite3.Binary(address))).fetchall()] + for txi in tx.inputs: txoid = t.execute( - "SELECT txoid, address FROM txo WHERE txid = ? AND position = ?", - (sqlite3.Binary(txi.output_txid), txi.output_index) + "SELECT txoid FROM txo WHERE txhash = ? AND position = ?", + (sqlite3.Binary(txi.output_txhash), txi.output_index) ).fetchone() - if txoid: + if txoid is not None and txoid[0] not in existing_txis: t.execute(*self._insert_sql("txi", { - 'txid': sqlite3.Binary(tx.id), + 'txhash': sqlite3.Binary(tx.hash), 'address': sqlite3.Binary(address), - 'txoid': txoid, + 'txoid': txoid[0], })) + return self.db.runInteraction(_steps) @defer.inlineCallbacks def get_balance_for_account(self, account): result = yield self.db.runQuery( - "SELECT SUM(amount) FROM txo NATURAL JOIN pubkey_address WHERE account=:account AND " - "txoid NOT IN (SELECT txoid FROM txi)", + """ + SELECT SUM(amount) FROM txo + JOIN pubkey_address ON pubkey_address.address=txo.address + WHERE account=:account AND + txoid NOT IN (SELECT txoid FROM txi) + """, {'account': sqlite3.Binary(account.public_key.address)} ) if result: @@ -195,8 +217,8 @@ class BaseDatabase(SQLiteMixin): def get_utxos(self, account, output_class): utxos = yield self.db.runQuery( """ - SELECT amount, script, txid, position - FROM txo NATURAL JOIN pubkey_address + SELECT amount, script, txhash, txo.position + FROM txo JOIN pubkey_address ON pubkey_address.address=txo.address WHERE account=:account AND txoid NOT IN (SELECT txoid FROM txi) """, {'account': sqlite3.Binary(account.public_key.address)} diff --git a/torba/baseledger.py b/torba/baseledger.py index 1acc1c1a1..8c28d70bc 100644 --- a/torba/baseledger.py +++ b/torba/baseledger.py @@ -76,9 +76,6 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): raw_address = self.pubkey_address_prefix + h160 return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4])) - def account_created(self, account): # type: (baseaccount.BaseAccount) -> None - self.accounts.add(account) - @staticmethod def address_to_hash160(address): bytes = Base58.decode(address) @@ -108,10 +105,16 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): def add_transaction(self, address, transaction, height): # type: (bytes, basetransaction.BaseTransaction, int) -> None yield self.db.add_transaction( - address, self.address_to_hash160(address), transaction, height, False, False + address, self.address_to_hash160(address), transaction, height, False ) self._on_transaction_controller.add(transaction) + @defer.inlineCallbacks + def add_account(self, account): # type: (baseaccount.BaseAccount) -> None + self.accounts.add(account) + if self.network.is_connected: + yield self.update_account(account) + @defer.inlineCallbacks def get_private_key_for_address(self, address): match = yield self.db.get_address(address) @@ -123,19 +126,6 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): def get_unspent_outputs(self, account): return self.db.get_utxos(account, self.transaction_class.output_class) -# def get_unspent_outputs(self, account): -# inputs, outputs, utxos = set(), set(), set() -# for address in self.addresses.values(): -# for tx in address: -# for txi in tx.inputs: -# inputs.add((hexlify(txi.output_txid), txi.output_index)) -# for txo in tx.outputs: -# if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == address.pubkey_hash: -# outputs.add((txo, txo.transaction.id, txo.index)) -# for output in outputs: -# if output[1:] not in inputs: -# yield output[0] - @defer.inlineCallbacks def get_local_status(self, address): address_details = yield self.db.get_address(address) @@ -146,6 +136,8 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)): def get_local_history(self, address): address_details = yield self.db.get_address(address) history = address_details['history'] or b'' + if six.PY2: + history = str(history) parts = history.split(b':')[:-1] defer.returnValue(list(zip(parts[0::2], map(int, parts[1::2])))) diff --git a/torba/basenetwork.py b/torba/basenetwork.py index 2b16ee767..509eea87f 100644 --- a/torba/basenetwork.py +++ b/torba/basenetwork.py @@ -95,7 +95,7 @@ class StratumClientProtocol(LineOnlyReceiver): try: d = self.lookup_table.pop(message['id']) if message.get('error'): - d.errback(RuntimeError(*message['error'])) + d.errback(RuntimeError(message['error'])) else: d.callback(message.get('result')) except KeyError: diff --git a/torba/basetransaction.py b/torba/basetransaction.py index 54e46cb7a..91e43efa4 100644 --- a/torba/basetransaction.py +++ b/torba/basetransaction.py @@ -5,12 +5,13 @@ from binascii import hexlify from twisted.internet import defer +import torba.baseaccount +import torba.baseledger from torba.basescript import BaseInputScript, BaseOutputScript from torba.coinselection import CoinSelector from torba.constants import COIN from torba.bcd_data_stream import BCDataStream from torba.hash import sha256 -from torba.baseaccount import BaseAccount from torba.util import ReadOnlyList @@ -22,16 +23,16 @@ NULL_HASH = b'\x00'*32 class InputOutput(object): - def __init__(self, txid, index=None): - self._txid = txid # type: bytes + def __init__(self, txhash, index=None): + self._txhash = txhash # type: bytes self.transaction = None # type: BaseTransaction self.index = index # type: int @property - def txid(self): - if self._txid is None: - self._txid = self.transaction.id - return self._txid + def txhash(self): + if self._txhash is None: + self._txhash = self.transaction.hash + return self._txhash @property def size(self): @@ -51,26 +52,20 @@ class BaseInput(InputOutput): NULL_SIGNATURE = b'\x00'*72 NULL_PUBLIC_KEY = b'\x00'*33 - def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF, txid=None): - super(BaseInput, self).__init__(txid) - if isinstance(output_or_txid_index, BaseOutput): - self.output = output_or_txid_index # type: BaseOutput - self.output_txid = self.output.txid + def __init__(self, output_or_txhash_index, script, sequence=0xFFFFFFFF, txhash=None): + super(BaseInput, self).__init__(txhash) + if isinstance(output_or_txhash_index, BaseOutput): + self.output = output_or_txhash_index # type: BaseOutput + self.output_txhash = self.output.txhash self.output_index = self.output.index else: self.output = None # type: BaseOutput - self.output_txid, self.output_index = output_or_txid_index + self.output_txhash, self.output_index = output_or_txhash_index self.sequence = sequence - self.is_coinbase = self.output_txid == NULL_HASH + self.is_coinbase = self.output_txhash == NULL_HASH self.coinbase = script if self.is_coinbase else None self.script = script if not self.is_coinbase else None # type: BaseInputScript - def link_output(self, output): - assert self.output is None - assert self.output_txid == output.transaction.id - assert self.output_index == output.index - self.output = output - @classmethod def spend(cls, output): """ Create an input to spend the output.""" @@ -87,18 +82,18 @@ class BaseInput(InputOutput): @classmethod def deserialize_from(cls, stream): - txid = stream.read(32) + txhash = stream.read(32) index = stream.read_uint32() script = stream.read_string() sequence = stream.read_uint32() return cls( - (txid, index), - cls.script_class(script) if not txid == NULL_HASH else script, + (txhash, index), + cls.script_class(script) if not txhash == NULL_HASH else script, sequence ) def serialize_to(self, stream, alternate_script=None): - stream.write(self.output_txid) + stream.write(self.output_txhash) stream.write_uint32(self.output_index) if alternate_script is not None: stream.write_string(alternate_script) @@ -114,7 +109,7 @@ class BaseOutputEffectiveAmountEstimator(object): __slots__ = 'coin', 'txi', 'txo', 'fee', 'effective_amount' - def __init__(self, ledger, txo): # type: (BaseLedger, BaseOutput) -> None + def __init__(self, ledger, txo): # type: (torba.baseledger.BaseLedger, BaseOutput) -> None self.txo = txo self.txi = ledger.transaction_class.input_class.spend(txo) self.fee = ledger.get_input_output_fee(self.txi) @@ -129,8 +124,8 @@ class BaseOutput(InputOutput): script_class = BaseOutputScript estimator_class = BaseOutputEffectiveAmountEstimator - def __init__(self, amount, script, txid=None, index=None): - super(BaseOutput, self).__init__(txid, index) + def __init__(self, amount, script, txhash=None, index=None): + super(BaseOutput, self).__init__(txhash, index) self.amount = amount # type: int self.script = script # type: BaseOutputScript @@ -259,7 +254,7 @@ class BaseTransaction: for txout in self._outputs: txout.serialize_to(stream) stream.write_uint32(self.locktime) - stream.write_uint32(1) # signature hash type: SIGHASH_ALL + stream.write_uint32(self.signature_hash_type(1)) # signature hash type: SIGHASH_ALL return stream.get_bytes() def _deserialize(self): @@ -279,7 +274,7 @@ class BaseTransaction: @classmethod @defer.inlineCallbacks def get_effective_amount_estimators(cls, funding_accounts): - # type: (Iterable[BaseAccount]) -> Generator[BaseOutputEffectiveAmountEstimator] + # type: (Iterable[torba.baseaccount.BaseAccount]) -> Generator[BaseOutputEffectiveAmountEstimator] estimators = [] for account in funding_accounts: utxos = yield account.ledger.get_unspent_outputs(account) @@ -289,7 +284,7 @@ class BaseTransaction: @classmethod def ensure_all_have_same_ledger(cls, funding_accounts, change_account=None): - # type: (Iterable[BaseAccount], BaseAccount) -> baseledger.BaseLedger + # type: (Iterable[torba.baseaccount.BaseAccount], torba.baseaccount.BaseAccount) -> torba.baseledger.BaseLedger ledger = None for account in funding_accounts: if ledger is None: @@ -305,12 +300,12 @@ class BaseTransaction: @classmethod @defer.inlineCallbacks def pay(cls, outputs, funding_accounts, change_account): - # type: (List[BaseOutput], List[BaseAccount], BaseAccount) -> BaseTransaction + # type: (List[BaseOutput], List[torba.baseaccount.BaseAccount], torba.baseaccount.BaseAccount) -> BaseTransaction """ Efficiently spend utxos from funding_accounts to cover the new outputs. """ tx = cls().add_outputs(outputs) ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account) - amount = ledger.get_transaction_base_fee(tx) + amount = tx.output_sum + ledger.get_transaction_base_fee(tx) txos = yield cls.get_effective_amount_estimators(funding_accounts) selector = CoinSelector( txos, amount, @@ -325,34 +320,38 @@ class BaseTransaction: spent_sum = sum(s.effective_amount for s in spendables) if spent_sum > amount: - change_address = change_account.change.get_or_create_usable_address() + change_address = yield change_account.change.get_or_create_usable_address() change_hash160 = change_account.ledger.address_to_hash160(change_address) change_amount = spent_sum - amount tx.add_outputs([cls.output_class.pay_pubkey_hash(change_amount, change_hash160)]) tx.add_inputs([s.txi for s in spendables]) - tx.sign(funding_accounts) + yield tx.sign(funding_accounts) defer.returnValue(tx) @classmethod def liquidate(cls, assets, funding_accounts, change_account): """ Spend assets (utxos) supplementing with funding_accounts if fee is higher than asset value. """ - def sign(self, funding_accounts): # type: (Iterable[BaseAccount]) -> BaseTransaction + def signature_hash_type(self, hash_type): + return hash_type + + @defer.inlineCallbacks + def sign(self, funding_accounts): # type: (Iterable[torba.baseaccount.BaseAccount]) -> BaseTransaction ledger = self.ensure_all_have_same_ledger(funding_accounts) for i, txi in enumerate(self._inputs): txo_script = txi.output.script if txo_script.is_pay_pubkey_hash: address = ledger.hash160_to_address(txo_script.values['pubkey_hash']) - private_key = ledger.get_private_key_for_address(address) + private_key = yield ledger.get_private_key_for_address(address) tx = self._serialize_for_signature(i) - txi.script.values['signature'] = private_key.sign(tx)+six.int2byte(1) + txi.script.values['signature'] = \ + private_key.sign(tx) + six.int2byte(self.signature_hash_type(1)) txi.script.values['pubkey'] = private_key.public_key.pubkey_bytes txi.script.generate() else: raise NotImplementedError("Don't know how to spend this output.") self._reset() - return self def sort(self): # See https://github.com/kristovatlas/rfc/blob/master/bips/bip-li01.mediawiki @@ -361,8 +360,8 @@ class BaseTransaction: @property def input_sum(self): - return sum(i.amount for i in self._inputs) + return sum(i.amount for i in self.inputs) @property def output_sum(self): - return sum(o.amount for o in self._outputs) + return sum(o.amount for o in self.outputs) diff --git a/torba/coin/bitcoincash.py b/torba/coin/bitcoincash.py index 1830f9cdb..ca9a76a41 100644 --- a/torba/coin/bitcoincash.py +++ b/torba/coin/bitcoincash.py @@ -4,63 +4,26 @@ __node_bin__ = 'bitcoin-abc-0.17.2/bin' __node_url__ = ( 'https://download.bitcoinabc.org/0.17.2/linux/bitcoin-abc-0.17.2-x86_64-linux-gnu.tar.gz' ) +__electrumx__ = 'electrumx.lib.coins.BitcoinCashRegtest' from six import int2byte from binascii import unhexlify -from torba.baseledger import BaseLedger, BaseHeaders -from torba.basenetwork import BaseNetwork -from torba.basescript import BaseInputScript, BaseOutputScript -from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput -from torba.basedatabase import BaseSQLiteWalletStorage -from torba.manager import BaseWalletManager - - -class WalletManager(BaseWalletManager): - pass - - -class Input(BaseInput): - script_class = BaseInputScript - - -class Output(BaseOutput): - script_class = BaseOutputScript +from torba.baseledger import BaseLedger +from torba.baseheader import BaseHeaders +from torba.basetransaction import BaseTransaction class Transaction(BaseTransaction): - input_class = Input - output_class = Output + + def signature_hash_type(self, hash_type): + return hash_type | 0x40 -class BitcoinCashLedger(BaseLedger): - network_class = BaseNetwork - headers_class = BaseHeaders - database_class = BaseSQLiteWalletStorage - - -class MainNetLedger(BitcoinCashLedger): - pass - - -class UnverifiedHeaders(BaseHeaders): - verify_bits_to_target = False - - -class RegTestLedger(BitcoinCashLedger): - headers_class = UnverifiedHeaders - max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206' - genesis_bits = 0x207fffff - target_timespan = 1 - verify_bits_to_target = False - - -class BitcoinCash(BaseCoin): +class MainNetLedger(BaseLedger): name = 'BitcoinCash' symbol = 'BCH' - network = 'mainnet' + network_name = 'mainnet' - ledger_class = MainNetLedger transaction_class = Transaction pubkey_address_prefix = int2byte(0x00) @@ -70,15 +33,21 @@ class BitcoinCash(BaseCoin): default_fee_per_byte = 50 - def __init__(self, ledger, fee_per_byte=default_fee_per_byte): - super(BitcoinCash, self).__init__(ledger, fee_per_byte) + +class UnverifiedHeaders(BaseHeaders): + verify_bits_to_target = False -class BitcoinCashRegtest(BitcoinCash): - network = 'regtest' - ledger_class = RegTestLedger +class RegTestLedger(MainNetLedger): + headers_class = UnverifiedHeaders + network_name = 'regtest' + pubkey_address_prefix = int2byte(111) script_address_prefix = int2byte(196) extended_public_key_prefix = unhexlify('043587cf') extended_private_key_prefix = unhexlify('04358394') + max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206' + genesis_bits = 0x207fffff + target_timespan = 1 diff --git a/torba/coin/bitcoinsegwit.py b/torba/coin/bitcoinsegwit.py index 6739d87f2..68301a57a 100644 --- a/torba/coin/bitcoinsegwit.py +++ b/torba/coin/bitcoinsegwit.py @@ -4,6 +4,7 @@ __node_bin__ = 'bitcoin-0.16.0/bin' __node_url__ = ( 'https://bitcoin.org/bin/bitcoin-core-0.16.0/bitcoin-0.16.0-x86_64-linux-gnu.tar.gz' ) +__electrumx__ = 'electrumx.lib.coins.BitcoinSegwitRegtest' from six import int2byte from binascii import unhexlify diff --git a/torba/hash.py b/torba/hash.py index 243ecdcde..3d7092e58 100644 --- a/torba/hash.py +++ b/torba/hash.py @@ -120,6 +120,9 @@ class Base58(object): @classmethod def decode(cls, txt): """ Decodes txt into a big-endian bytearray. """ + if six.PY2 and isinstance(txt, buffer): + txt = str(txt) + if isinstance(txt, six.binary_type): txt = txt.decode() diff --git a/torba/wallet.py b/torba/wallet.py index 38421c362..a93b77f25 100644 --- a/torba/wallet.py +++ b/torba/wallet.py @@ -1,17 +1,10 @@ import stat import json import os -from typing import List, Dict +from typing import List -from torba.baseaccount import BaseAccount -from torba.baseledger import LedgerRegistry, BaseLedger - - -def inflate_ledger(manager, ledger_id, ledger_dict): - # type: ('WalletManager', str, Dict) -> BaseLedger - ledger_class = LedgerRegistry.get_ledger_class(ledger_id) - ledger = manager.get_or_create_ledger(ledger_id) - return ledger_class(ledger, **ledger_dict) +import torba.baseaccount +import torba.baseledger class Wallet: @@ -21,13 +14,14 @@ class Wallet: by physical files on the filesystem. """ - def __init__(self, name='Wallet', ledgers=None, accounts=None, storage=None): + def __init__(self, name='Wallet', accounts=None, storage=None): + # type: (str, List[torba.baseaccount.BaseAccount], WalletStorage) -> None self.name = name - self.ledgers = ledgers or [] # type: List[BaseLedger] - self.accounts = accounts or [] # type: List[BaseAccount] + self.accounts = accounts or [] self.storage = storage or WalletStorage() - def generate_account(self, ledger): # type: (BaseLedger) -> Account + def generate_account(self, ledger): + # type: (torba.baseledger.BaseLedger) -> torba.baseaccount.BaseAccount account = ledger.account_class.generate(ledger, u'torba') self.accounts.append(account) return account @@ -36,22 +30,14 @@ class Wallet: def from_storage(cls, storage, manager): # type: (WalletStorage, 'WalletManager') -> Wallet json_dict = storage.read() - ledgers = {} - for ledger_id, ledger_dict in json_dict.get('ledgers', {}).items(): - ledgers[ledger_id] = inflate_ledger(manager, ledger_id, ledger_dict) - accounts = [] for account_dict in json_dict.get('accounts', []): - ledger_id = account_dict['ledger'] - ledger = ledgers.get(ledger_id) - if ledger is None: - ledger = ledgers[ledger_id] = inflate_ledger(manager, ledger_id, {}) + ledger = manager.get_or_create_ledger(account_dict['ledger']) account = ledger.account_class.from_dict(ledger, account_dict) accounts.append(account) return cls( name=json_dict.get('name', 'Wallet'), - ledgers=list(ledgers.values()), accounts=accounts, storage=storage ) @@ -59,7 +45,6 @@ class Wallet: def to_dict(self): return { 'name': self.name, - 'ledgers': {c.get_id(): {} for c in self.ledgers}, 'accounts': [a.to_dict() for a in self.accounts] } @@ -71,12 +56,6 @@ class Wallet: for account in self.accounts: return account - def get_account_private_key_for_address(self, address): - for account in self.accounts: - private_key = account.get_private_key_for_address(address) - if private_key is not None: - return account, private_key - class WalletStorage: diff --git a/tox.ini b/tox.ini index 7099b9dd7..798e9778b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,14 @@ [tox] -envlist = py{27,36} +envlist = py27,py36-{torba.coin.bitcoincash,torba.coin.bitcoinsegwit} [testenv] deps = coverage - mock - + py36: ../orchstr8 + py36: ../electrumx +extras = test changedir = {toxinidir}/tests - +setenv = py36: LEDGER={envname} commands = - coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -v + coverage run -p --source={envsitepackagesdir}/torba -m twisted.trial unit + py36: coverage run -p --source={envsitepackagesdir}/torba -m twisted.trial --reactor=asyncio integration