diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_transactions.py b/tests/integration/test_transactions.py new file mode 100644 index 000000000..fd16cc534 --- /dev/null +++ b/tests/integration/test_transactions.py @@ -0,0 +1,32 @@ +import asyncio +from orchstr8.testcase import IntegrationTestCase +from torba.constants import COIN + + +class BasicTransactionTests(IntegrationTestCase): + + VERBOSE = True + + async def test_sending_and_recieving(self): + account1, account2 = self.account, self.wallet.generate_account(self.ledger) + + self.assertEqual(await self.get_balance(account1), 0) + self.assertEqual(await self.get_balance(account2), 0) + + address = await account1.get_least_used_receiving_address().asFuture(asyncio.get_event_loop()) + sendtxid = await self.blockchain.send_to_address(address.decode(), 5.5) + await self.blockchain.generate(1) + await self.on_transaction(sendtxid) + + self.assertEqual(await self.get_balance(account1), int(5.5*COIN)) + self.assertEqual(await self.get_balance(account2), 0) + + address = await account2.get_least_used_receiving_address().asFuture(asyncio.get_event_loop()) + sendtxid = await self.blockchain.send_to_address(address.decode(), 5.5) + await self.broadcast(tx) + await self.on_transaction(tx.id.decode()) + await self.lbrycrd.generate(1) + + self.assertEqual(await self.get_balance(account1), int(3.0*COIN)) + self.assertEqual(await self.get_balance(account2), int(2.5*COIN)) + diff --git a/tests/unit/test_account.py b/tests/unit/test_account.py index 3648c5fe4..b75eb9b31 100644 --- a/tests/unit/test_account.py +++ b/tests/unit/test_account.py @@ -1,8 +1,8 @@ from binascii import hexlify from twisted.trial import unittest -from torba.coin.btc import BTC -from torba.manager import WalletManager +from torba.coin.bitcoinsegwit import BTC +from torba.basemanager import WalletManager from torba.wallet import Account diff --git a/tests/unit/test_coinselection.py b/tests/unit/test_coinselection.py index d57ff172d..4e2c2b565 100644 --- a/tests/unit/test_coinselection.py +++ b/tests/unit/test_coinselection.py @@ -1,9 +1,9 @@ import unittest -from torba.coin.btc import BTC +from torba.coin.bitcoinsegwit import BTC from torba.coinselection import CoinSelector, MAXIMUM_TRIES from torba.constants import CENT -from torba.manager import WalletManager +from torba.basemanager import WalletManager from .test_transaction import Output, get_output as utxo diff --git a/tests/unit/test_transaction.py b/tests/unit/test_transaction.py index fcdd2d736..bf34afc20 100644 --- a/tests/unit/test_transaction.py +++ b/tests/unit/test_transaction.py @@ -2,9 +2,9 @@ from binascii import hexlify, unhexlify from twisted.trial import unittest from torba.account import Account -from torba.coin.btc import BTC, Transaction, Output, Input +from torba.coin.bitcoinsegwit import BTC, Transaction, Output, Input from torba.constants import CENT, COIN -from torba.manager import WalletManager +from torba.basemanager import WalletManager from torba.wallet import Wallet diff --git a/tests/unit/test_wallet.py b/tests/unit/test_wallet.py index 09d0bf884..52e1240dd 100644 --- a/tests/unit/test_wallet.py +++ b/tests/unit/test_wallet.py @@ -1,7 +1,7 @@ from twisted.trial import unittest -from torba.coin.btc import BTC -from torba.manager import WalletManager +from torba.coin.bitcoinsegwit import BTC +from torba.basemanager import WalletManager from torba.wallet import Account, Wallet, WalletStorage from .ftc import FTC diff --git a/torba/account.py b/torba/account.py index b166fcd3b..acc518b5c 100644 --- a/torba/account.py +++ b/torba/account.py @@ -1,6 +1,7 @@ import itertools from typing import Dict, Generator from binascii import hexlify, unhexlify +from twisted.internet import defer from torba.basecoin import BaseCoin from torba.mnemonic import Mnemonic @@ -20,14 +21,14 @@ class KeyChain: for key in child_keys ] - @property + @defer.inlineCallbacks def has_gap(self): if len(self.addresses) < self.minimum_gap: - return False + defer.returnValue(False) for address in self.addresses[-self.minimum_gap:]: - if self.coin.ledger.is_address_old(address): - return False - return True + if (yield self.coin.ledger.is_address_old(address)): + defer.returnValue(False) + defer.returnValue(True) def generate_next_address(self): child_key = self.parent_key.child(len(self.child_keys)) @@ -35,11 +36,12 @@ class KeyChain: self.addresses.append(child_key.address) return child_key.address + @defer.inlineCallbacks def ensure_enough_addresses(self): starting_length = len(self.addresses) - while not self.has_gap: + while not (yield self.has_gap()): self.generate_next_address() - return self.addresses[starting_length:] + defer.returnValue(self.addresses[starting_length:]) class Account: @@ -135,50 +137,39 @@ class Account: if address == match: return self.private_key.child(a).child(b) + @defer.inlineCallbacks def ensure_enough_addresses(self): - return [ - address - for keychain in self.keychains - for address in keychain.ensure_enough_addresses() - ] - - def addresses_without_history(self): - for address in self.addresses: - if not self.coin.ledger.has_address(address): - yield address + addresses = [] + for keychain in self.keychains: + for address in (yield keychain.ensure_enough_addresses()): + addresses.append(address) + defer.returnValue(addresses) def get_least_used_receiving_address(self, max_transactions=1000): return self._get_least_used_address( - self.receiving_keys.addresses, self.receiving_keys, max_transactions ) def get_least_used_change_address(self, max_transactions=100): return self._get_least_used_address( - self.change_keys.addresses, self.change_keys, max_transactions ) - def _get_least_used_address(self, addresses, keychain, max_transactions): + def _get_least_used_address(self, keychain, max_transactions): ledger = self.coin.ledger - address = ledger.get_least_used_address(addresses, max_transactions) + address = ledger.get_least_used_address(self, keychain, max_transactions) if address: return address address = keychain.generate_next_address() ledger.subscribe_history(address) return address - def get_unspent_utxos(self): - return [ - utxo - for address in self.addresses - for utxo in self.coin.ledger.get_unspent_outputs(address) - ] - + @defer.inlineCallbacks def get_balance(self): - return sum(utxo.amount for utxo in self.get_unspent_utxos()) + utxos = yield self.coin.ledger.get_unspent_outputs(self) + defer.returnValue(sum(utxo.amount for utxo in utxos)) class AccountsView: @@ -188,3 +179,13 @@ class AccountsView: def __iter__(self): # type: () -> Generator[Account] return self._accounts_generator() + + def addresses(self): + for account in self: + for address in account.addresses: + yield address + + def get_account_for_address(self, address): + for account in self: + if address in account.addresses: + return account \ No newline at end of file diff --git a/torba/basecoin.py b/torba/basecoin.py index 2b8c04502..7b00115cd 100644 --- a/torba/basecoin.py +++ b/torba/basecoin.py @@ -44,22 +44,13 @@ class BaseCoin(six.with_metaclass(CoinRegistry)): def __init__(self, ledger, fee_per_byte): self.ledger = ledger - self.fee_per_byte = fee_per_byte @classmethod def get_id(cls): return '{}_{}'.format(cls.symbol.lower(), cls.network.lower()) def to_dict(self): - return {'fee_per_byte': self.fee_per_byte} - - def get_input_output_fee(self, io): - """ Fee based on size of the input / output. """ - return self.fee_per_byte * io.size - - def get_transaction_base_fee(self, tx): - """ Fee for the transaction header and all outputs; without inputs. """ - return self.fee_per_byte * tx.base_size + return {} def hash160_to_address(self, h160): raw_address = self.pubkey_address_prefix + h160 diff --git a/torba/basedatabase.py b/torba/basedatabase.py new file mode 100644 index 000000000..1b8082690 --- /dev/null +++ b/torba/basedatabase.py @@ -0,0 +1,219 @@ +import logging +import os +import sqlite3 +from twisted.internet import defer +from twisted.enterprise import adbapi + +log = logging.getLogger(__name__) + + +class BaseSQLiteWalletStorage(object): + + CREATE_TX_TABLE = """ + create table if not exists tx ( + txid blob primary key, + raw blob not null, + height integer not null, + is_confirmed boolean not null, + is_verified boolean not null + ); + create table if not exists address_status ( + address blob not null, + status text not null + ); + """ + + CREATE_TXO_TABLE = """ + create table if not exists txo ( + txoid integer primary key, + account blob not null, + address blob not null, + txid blob references tx, + pos integer not null, + amount integer not null, + script blob not null + ); + """ + + CREATE_TXI_TABLE = """ + create table if not exists txi ( + account blob not null, + txid blob references tx, + txoid integer references txo + ); + """ + + CREATE_TABLES_QUERY = ( + CREATE_TX_TABLE + + CREATE_TXO_TABLE + + CREATE_TXI_TABLE + ) + + def __init__(self, ledger): + self._db_path = os.path.join(ledger.path, "blockchain.db") + self.db = None + + def start(self): + log.info("connecting to database: %s", self._db_path) + self.db = adbapi.ConnectionPool( + 'sqlite3', self._db_path, cp_min=1, cp_max=1, check_same_thread=False + ) + return self.db.runInteraction( + lambda t: t.executescript(self.CREATE_TABLES_QUERY) + ) + + def stop(self): + self.db.close() + return defer.succeed(True) + + @defer.inlineCallbacks + def run_and_return_one_or_none(self, query, *args): + result = yield self.db.runQuery(query, args) + if result: + defer.returnValue(result[0][0]) + else: + defer.returnValue(None) + + @defer.inlineCallbacks + def run_and_return_list(self, query, *args): + result = yield self.db.runQuery(query, args) + if result: + defer.returnValue([i[0] for i in result]) + else: + defer.returnValue([]) + + def run_and_return_id(self, query, *args): + def do_save(t): + t.execute(query, args) + return t.lastrowid + return self.db.runInteraction(do_save) + + def add_transaction(self, tx, height, is_confirmed, is_verified): + return self.run_and_return_id( + "insert into tx values (?, ?, ?, ?, ?)", + sqlite3.Binary(tx.id), + sqlite3.Binary(tx.raw), + height, + is_confirmed, + is_verified + ) + + @defer.inlineCallbacks + def has_transaction(self, txid): + result = yield self.db.runQuery( + "select rowid from tx where txid=?", (txid,) + ) + defer.returnValue(bool(result)) + + def add_tx_output(self, account, txo): + return self.db.runOperation( + "insert into txo values (?, ?, ?, ?, ?, ?, ?, ?, ?)", ( + sqlite3.Binary(account.public_key.address), + sqlite3.Binary(txo.script.values['pubkey_hash']), + sqlite3.Binary(txo.txid), + txo.index, + txo.amount, + sqlite3.Binary(txo.script.source), + txo.script.is_claim_name, + txo.script.is_support_claim, + txo.script.is_update_claim + ) + ) + + def add_tx_input(self, account, txi): + def _ops(t): + txoid = t.execute( + "select rowid from txo where txid=? and pos=?", ( + sqlite3.Binary(txi.output_txid), txi.output_index + ) + ).fetchone()[0] + t.execute( + "insert into txi values (?, ?, ?)", ( + sqlite3.Binary(account.public_key.address), + sqlite3.Binary(txi.txid), + txoid + ) + ) + return self.db.runInteraction(_ops) + + @defer.inlineCallbacks + def get_balance_for_account(self, account): + result = yield self.db.runQuery( + "select sum(amount) from txo where account=:account and rowid not in (select txo from txi where account=:account)", + {'account': sqlite3.Binary(account.public_key.address)} + ) + if result: + defer.returnValue(result[0][0] or 0) + else: + defer.returnValue(0) + + def get_used_addresses(self, account): + return self.db.runQuery( + """ + SELECT + txios.address, + sum(txios.used_count) as total + FROM + (SELECT address, count(*) as used_count FROM txo + WHERE account=:account GROUP BY address + UNION + SELECT address, count(*) as used_count FROM txi NATURAL JOIN txo + WHERE account=:account GROUP BY address) AS txios + GROUP BY txios.address + ORDER BY total + """, {'account': sqlite3.Binary(account.public_key.address)} + ) + + @defer.inlineCallbacks + def get_earliest_block_height_for_address(self, address): + result = yield self.db.runQuery( + """ + SELECT + height + FROM + (SELECT DISTINCT height FROM txi NATURAL JOIN txo NATURAL JOIN tx WHERE address=:address + UNION + SELECT DISTINCT height FROM txo NATURAL JOIN tx WHERE address=:address) AS txios + ORDER BY height LIMIT 1 + """, {'address': sqlite3.Binary(address)} + ) + if result: + defer.returnValue(result[0][0]) + else: + defer.returnValue(None) + + @defer.inlineCallbacks + def get_utxos(self, account, output_class): + utxos = yield self.db.runQuery( + """ + SELECT + amount, script, txid + FROM txo + WHERE + account=:account AND + txoid NOT IN (SELECT txoid FROM txi WHERE account=:account) + """, + {'account': sqlite3.Binary(account.public_key.address)} + ) + defer.returnValue([ + output_class( + values[0], + output_class.script_class(values[1]), + values[2] + ) for values in utxos + ]) + + @defer.inlineCallbacks + def get_address_status(self, address): + result = yield self.db.runQuery( + "select status from address_status where address=?", (address,) + ) + if result: + defer.returnValue(result[0][0]) + else: + defer.returnValue(None) + + def set_address_status(self, address, status): + return self.db.runOperation( + "replace into address_status (address, status) values (?, ?)", (address,status) + ) diff --git a/torba/baseledger.py b/torba/baseledger.py index 99fb5f1d1..9f913df38 100644 --- a/torba/baseledger.py +++ b/torba/baseledger.py @@ -1,17 +1,18 @@ import os import hashlib +import struct from binascii import hexlify, unhexlify from typing import List, Dict, Type from operator import itemgetter from twisted.internet import threads, defer, task, reactor +from torba import basetransaction, basedatabase from torba.account import Account, AccountsView from torba.basecoin import BaseCoin -from torba.basetransaction import BaseTransaction from torba.basenetwork import BaseNetwork from torba.stream import StreamController, execute_serially -from torba.util import hex_to_int, int_to_hex, rev_hex, hash_encode +from torba.util import int_to_hex, rev_hex, hash_encode from torba.hash import double_sha256, pow_hash @@ -28,22 +29,11 @@ class Address: return len(self.transactions) def add_transaction(self, transaction): - self.transactions.append(transaction) - - def get_unspent_utxos(self): - inputs, outputs, utxos = [], [], [] - for tx in self: - for txi in tx.inputs: - inputs.append((txi.output_txid, txi.output_index)) - for txo in tx.outputs: - if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == self.pubkey_hash: - outputs.append((txo, txo.transaction.hash, txo.index)) - for output in set(outputs): - if output[1:] not in inputs: - yield output[0] + if transaction not in self.transactions: + self.transactions.append(transaction) -class BaseLedger: +class BaseLedger(object): # coin_class is automatically set by BaseCoin metaclass # when it creates the Coin classes, there is a 1..1 relationship @@ -52,21 +42,38 @@ class BaseLedger: # but many coin instances can exist linking back to the single Ledger instance. coin_class = None # type: Type[BaseCoin] network_class = None # type: Type[BaseNetwork] + headers_class = None # type: Type[BaseHeaders] + database_class = None # type: Type[basedatabase.BaseSQLiteWalletStorage] - verify_bits_to_target = True + default_fee_per_byte = 10 - def __init__(self, accounts, config=None, network=None, db=None): + def __init__(self, accounts, config=None, db=None, network=None, + fee_per_byte=default_fee_per_byte): self.accounts = accounts # type: AccountsView self.config = config or {} - self.db = db - self.addresses = {} # type: Dict[str, Address] - self.transactions = {} # type: Dict[str, BaseTransaction] - self.headers = Headers(self) - self._on_transaction_controller = StreamController() - self.on_transaction = self._on_transaction_controller.stream - self.network = network or self.network_class(self.config) + self.db = db or self.database_class(self) # type: basedatabase.BaseSQLiteWalletStorage + self.network = network or self.network_class(self) self.network.on_header.listen(self.process_header) self.network.on_status.listen(self.process_status) + self.headers = self.headers_class(self) + self.fee_per_byte = fee_per_byte + + self._on_transaction_controller = StreamController() + self.on_transaction = self._on_transaction_controller.stream + + @property + def path(self): + return os.path.join( + self.config['wallet_path'], self.coin_class.get_id() + ) + + def get_input_output_fee(self, io): + """ Fee based on size of the input / output. """ + return self.fee_per_byte * io.size + + def get_transaction_base_fee(self, tx): + """ Fee for the transaction header and all outputs; without inputs. """ + return self.fee_per_byte * tx.base_size @property def transaction_class(self): @@ -77,79 +84,51 @@ class BaseLedger: return cls(json_dict) @defer.inlineCallbacks - def load(self): - txs = yield self.db.get_transactions() - for tx_hash, raw, height in txs: - self.transactions[tx_hash] = self.transaction_class(raw, height) - txios = yield self.db.get_transaction_inputs_and_outputs() - for tx_hash, address_hash, input_output, amount, height in txios: - tx = self.transactions[tx_hash] - address = self.addresses.get(address_hash) - if address is None: - address = self.addresses[address_hash] = Address(self.coin_class.address_to_hash160(address_hash)) - tx.add_txio(address, input_output, amount) - address.add_transaction(tx) - def is_address_old(self, address, age_limit=2): - age = -1 - for tx in self.get_transactions(address, []): - if tx.height == 0: - tx_age = 0 - else: - tx_age = self.headers.height - tx.height + 1 - if tx_age > age: - age = tx_age + height = yield self.db.get_earliest_block_height_for_address(address) + if height is None: + return False + age = self.headers.height - height + 1 return age > age_limit - def add_transaction(self, address, transaction): # type: (str, BaseTransaction) -> None - if address not in self.addresses: - self.addresses[address] = Address(self.coin_class.address_to_hash160(address)) - self.addresses[address].add_transaction(transaction) - self.transactions.setdefault(transaction.id, transaction) + @defer.inlineCallbacks + def add_transaction(self, transaction, height): # type: (basetransaction.BaseTransaction, int) -> None + yield self.db.add_transaction(transaction, height, False, False) self._on_transaction_controller.add(transaction) def has_address(self, address): - return address in self.addresses + return address in self.accounts.addresses - def get_transaction(self, tx_hash, *args): - return self.transactions.get(tx_hash, *args) + @defer.inlineCallbacks + def get_least_used_address(self, account, keychain, max_transactions=100): + used_addresses = yield self.db.get_used_addresses(account) + unused_set = set(keychain.addresses) - set(map(itemgetter(0), used_addresses)) + if unused_set: + defer.returnValue(unused_set.pop()) + if used_addresses and used_addresses[0][1] < max_transactions: + defer.returnValue(used_addresses[0][0]) - def get_transactions(self, address, *args): - return self.addresses.get(address, *args) + def get_unspent_outputs(self, account): + return self.db.get_utxos(account, self.transaction_class.output_class) - def get_status(self, address): - hashes = [ - '{}:{}:'.format(hexlify(tx.hash), tx.height).encode() - for tx in self.get_transactions(address, []) if tx.height is not None - ] - if hashes: - return hexlify(hashlib.sha256(b''.join(hashes)).digest()) - - def has_transaction(self, tx_hash): - return tx_hash in self.transactions - - def get_least_used_address(self, addresses, max_transactions=100): - transaction_counts = [] - for address in addresses: - transactions = self.get_transactions(address, []) - tx_count = len(transactions) - if tx_count == 0: - return address - elif tx_count >= max_transactions: - continue - else: - transaction_counts.append((address, tx_count)) - if transaction_counts: - transaction_counts.sort(key=itemgetter(1)) - return transaction_counts[0] - - def get_unspent_outputs(self, address): - if address in self.addresses: - return list(self.addresses[address].get_unspent_utxos()) - return [] +# def get_unspent_outputs(self, account): +# inputs, outputs, utxos = set(), set(), set() +# for address in self.addresses.values(): +# for tx in address: +# for txi in tx.inputs: +# inputs.add((hexlify(txi.output_txid), txi.output_index)) +# for txo in tx.outputs: +# if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == address.pubkey_hash: +# outputs.add((txo, txo.transaction.id, txo.index)) +# for output in outputs: +# if output[1:] not in inputs: +# yield output[0] @defer.inlineCallbacks def start(self): + if not os.path.exists(self.path): + os.mkdir(self.path) + yield self.db.start() first_connection = self.network.on_connected.first self.network.start() yield first_connection @@ -197,13 +176,14 @@ class BaseLedger: # this avoids situation where we're getting status updates to addresses we know # need to update anyways. Continue to get history and create more addresses until # all missing addresses are created and history for them is fully restored. - account.ensure_enough_addresses() - addresses = list(account.addresses_without_history()) + yield account.ensure_enough_addresses() + used_addresses = yield self.db.get_used_addresses(account) + addresses = set(account.addresses) - set(map(itemgetter(0), used_addresses)) while addresses: yield defer.DeferredList([ self.update_history(a) for a in addresses ]) - addresses = account.ensure_enough_addresses() + addresses = yield account.ensure_enough_addresses() # By this point all of the addresses should be restored and we # can now subscribe all of them to receive updates. @@ -212,32 +192,49 @@ class BaseLedger: for address in account.addresses ]) + def _get_status_from_history(self, history): + hashes = [ + '{}:{}:'.format(hash.decode(), height).encode() + for hash, height in map(itemgetter('tx_hash', 'height'), history) + ] + if hashes: + return hexlify(hashlib.sha256(b''.join(hashes)).digest()) + @defer.inlineCallbacks - def update_history(self, address): + def update_history(self, address, remote_status=None): history = yield self.network.get_history(address) - for hash in map(itemgetter('tx_hash'), history): - transaction = self.get_transaction(hash) - if not transaction: + for hash, height in map(itemgetter('tx_hash', 'height'), history): + if not (yield self.db.has_transaction(hash)): raw = yield self.network.get_transaction(hash) transaction = self.transaction_class(unhexlify(raw)) - self.add_transaction(address, transaction) + yield self.add_transaction(transaction, height) + if remote_status is None: + remote_status = self._get_status_from_history(history) + if remote_status: + yield self.db.set_address_status(address, remote_status) @defer.inlineCallbacks def subscribe_history(self, address): - status = yield self.network.subscribe_address(address) - if status != self.get_status(address): - yield self.update_history(address) + remote_status = yield self.network.subscribe_address(address) + local_status = yield self.db.get_address_status(address) + if local_status != remote_status: + yield self.update_history(address, remote_status) + @defer.inlineCallbacks def process_status(self, response): - address, status = response - if status != self.get_status(address): - task.deferLater(reactor, 0, self.update_history, address) + address, remote_status = response + local_status = yield self.db.get_address_status(address) + if local_status != remote_status: + yield self.update_history(address, remote_status) def broadcast(self, tx): return self.network.broadcast(hexlify(tx.raw)) -class Headers: +class BaseHeaders: + + header_size = 80 + verify_bits_to_target = True def __init__(self, ledger): self.ledger = ledger @@ -247,9 +244,7 @@ class Headers: @property def path(self): - wallet_path = self.ledger.config.get('wallet_path', '') - filename = '{}_headers'.format(self.ledger.coin_class.get_id()) - return os.path.join(wallet_path, filename) + return os.path.join(self.ledger.path, 'headers') def touch(self): if not os.path.exists(self.path): @@ -261,13 +256,13 @@ class Headers: return len(self) - 1 def sync_read_length(self): - return os.path.getsize(self.path) // self.ledger.header_size + return os.path.getsize(self.path) // self.header_size def sync_read_header(self, height): if 0 <= height < len(self): with open(self.path, 'rb') as f: - f.seek(height * self.ledger.header_size) - return f.read(self.ledger.header_size) + f.seek(height * self.header_size) + return f.read(self.header_size) def __len__(self): if self._size is None: @@ -295,7 +290,7 @@ class Headers: previous_header = header with open(self.path, 'r+b') as f: - f.seek(start * self.ledger.header_size) + f.seek(start * self.header_size) f.write(headers) f.truncate() @@ -306,9 +301,9 @@ class Headers: self._on_change_controller.add(change) def _iterate_headers(self, height, headers): - assert len(headers) % self.ledger.header_size == 0 - for idx in range(len(headers) // self.ledger.header_size): - start, end = idx * self.ledger.header_size, (idx + 1) * self.ledger.header_size + assert len(headers) % self.header_size == 0 + for idx in range(len(headers) // self.header_size): + start, end = idx * self.header_size, (idx + 1) * self.header_size header = headers[start:end] yield self._deserialize(height+idx, header) @@ -317,15 +312,16 @@ class Headers: assert previous_hash == header['prev_block_hash'], \ "prev hash mismatch: {} vs {}".format(previous_hash, header['prev_block_hash']) - bits, target = self._calculate_lbry_next_work_required(height, previous_header, header) + bits, target = self._calculate_next_work_required(height, previous_header, header) assert bits == header['bits'], \ "bits mismatch: {} vs {} (hash: {})".format( bits, header['bits'], self._hash_header(header)) - _pow_hash = self._pow_hash_header(header) - assert int(b'0x' + _pow_hash, 16) <= target, \ - "insufficient proof of work: {} vs target {}".format( - int(b'0x' + _pow_hash, 16), target) + # TODO: FIX ME!!! + #_pow_hash = self._pow_hash_header(header) + #assert int(b'0x' + _pow_hash, 16) <= target, \ + # "insufficient proof of work: {} vs target {}".format( + # int(b'0x' + _pow_hash, 16), target) @staticmethod def _serialize(header): @@ -333,7 +329,6 @@ class Headers: int_to_hex(header['version'], 4), rev_hex(header['prev_block_hash']), rev_hex(header['merkle_root']), - rev_hex(header['claim_trie_root']), int_to_hex(int(header['timestamp']), 4), int_to_hex(int(header['bits']), 4), int_to_hex(int(header['nonce']), 4) @@ -341,15 +336,16 @@ class Headers: @staticmethod def _deserialize(height, header): + version, = struct.unpack('> 24) & 0xff - assert 0x03 <= bitsN <= 0x1f, \ + assert 0x03 <= bitsN <= 0x1d, \ "First part of bits should be in [0x03, 0x1d], but it was {}".format(hex(bitsN)) bitsBase = bits & 0xffffff assert 0x8000 <= bitsBase <= 0x7fffff, \ diff --git a/torba/manager.py b/torba/basemanager.py similarity index 83% rename from torba/manager.py rename to torba/basemanager.py index 84de427c1..ea5b6e9f7 100644 --- a/torba/manager.py +++ b/torba/basemanager.py @@ -5,13 +5,13 @@ from twisted.internet import defer from torba.account import AccountsView from torba.basecoin import CoinRegistry from torba.baseledger import BaseLedger -from torba.basetransaction import NULL_HASH +from torba.basetransaction import BaseTransaction, NULL_HASH from torba.coinselection import CoinSelector from torba.constants import COIN from torba.wallet import Wallet, WalletStorage -class WalletManager: +class BaseWalletManager(object): def __init__(self, wallets=None, ledgers=None): self.wallets = wallets or [] # type: List[Wallet] @@ -35,10 +35,22 @@ class WalletManager: ledger_class = coin_class.ledger_class ledger = self.ledgers.get(ledger_class) if ledger is None: - ledger = ledger_class(self.get_accounts_view(coin_class), ledger_config or {}) + ledger = self.create_ledger(ledger_class, self.get_accounts_view(coin_class), ledger_config or {}) self.ledgers[ledger_class] = ledger return ledger + def create_ledger(self, ledger_class, accounts, config): + return ledger_class(accounts, config) + + @defer.inlineCallbacks + def get_balance(self): + balances = {} + for ledger in self.ledgers: + for account in self.get_accounts(ledger.coin_class): + balances.setdefault(ledger.coin_class.name, 0) + balances[ledger.coin_class.name] += yield account.get_balance() + defer.returnValue(balances) + @property def default_wallet(self): for wallet in self.wallets: @@ -72,14 +84,14 @@ class WalletManager: return wallet.generate_account(ledger) @defer.inlineCallbacks - def start_ledgers(self): + def start(self): self.running = True yield defer.DeferredList([ l.start() for l in self.ledgers.values() ]) @defer.inlineCallbacks - def stop_ledgers(self): + def stop(self): yield defer.DeferredList([ l.stop() for l in self.ledgers.values() ]) @@ -91,12 +103,13 @@ class WalletManager: account = self.default_account coin = account.coin ledger = coin.ledger - tx_class = ledger.transaction_class + tx_class = ledger.transaction_class # type: BaseTransaction in_class, out_class = tx_class.input_class, tx_class.output_class estimators = [ txo.get_estimator(coin) for txo in account.get_unspent_utxos() ] + tx_class.create() cost_of_output = coin.get_input_output_fee( out_class.pay_pubkey_hash(COIN, NULL_HASH) diff --git a/torba/basenetwork.py b/torba/basenetwork.py index 59c2855e7..2b16ee767 100644 --- a/torba/basenetwork.py +++ b/torba/basenetwork.py @@ -137,8 +137,8 @@ class StratumClientFactory(protocol.ClientFactory): class BaseNetwork: - def __init__(self, config): - self.config = config + def __init__(self, ledger): + self.config = ledger.config self.client = None self.service = None self.running = False diff --git a/torba/basetransaction.py b/torba/basetransaction.py index 924186959..6963db4c0 100644 --- a/torba/basetransaction.py +++ b/torba/basetransaction.py @@ -1,10 +1,12 @@ import six import logging -from typing import List +from typing import List, Iterable, Generator from binascii import hexlify -from torba.basecoin import BaseCoin +from torba import baseledger from torba.basescript import BaseInputScript, BaseOutputScript +from torba.coinselection import CoinSelector +from torba.constants import COIN from torba.bcd_data_stream import BCDataStream from torba.hash import sha256 from torba.account import Account @@ -19,6 +21,17 @@ NULL_HASH = b'\x00'*32 class InputOutput(object): + def __init__(self, txid): + self._txid = txid # type: bytes + self.transaction = None # type: BaseTransaction + self.index = None # type: int + + @property + def txid(self): + if self._txid is None: + self._txid = self.transaction.id + return self._txid + @property def size(self): """ Size of this input / output in bytes. """ @@ -37,10 +50,11 @@ class BaseInput(InputOutput): NULL_SIGNATURE = b'\x00'*72 NULL_PUBLIC_KEY = b'\x00'*33 - def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF): + def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF, txid=None): + super(BaseInput, self).__init__(txid) if isinstance(output_or_txid_index, BaseOutput): self.output = output_or_txid_index # type: BaseOutput - self.output_txid = self.output.transaction.hash + self.output_txid = self.output.transaction.id self.output_index = self.output.index else: self.output = None # type: BaseOutput @@ -52,7 +66,7 @@ class BaseInput(InputOutput): def link_output(self, output): assert self.output is None - assert self.output_txid == output.transaction.hash + assert self.output_txid == output.transaction.id assert self.output_index == output.index self.output = output @@ -95,15 +109,14 @@ class BaseInput(InputOutput): stream.write_uint32(self.sequence) -class BaseOutputAmountEstimator(object): +class BaseOutputEffectiveAmountEstimator(object): __slots__ = 'coin', 'txi', 'txo', 'fee', 'effective_amount' - def __init__(self, coin, txo): # type: (BaseCoin, BaseOutput) -> None - self.coin = coin + def __init__(self, ledger, txo): # type: (baseledger.BaseLedger, BaseOutput) -> None self.txo = txo - self.txi = coin.transaction_class.input_class.spend(txo) - self.fee = coin.get_input_output_fee(self.txi) + self.txi = ledger.transaction_class.input_class.spend(txo) + self.fee = ledger.get_input_output_fee(self.txi) self.effective_amount = txo.amount - self.fee def __lt__(self, other): @@ -113,16 +126,15 @@ class BaseOutputAmountEstimator(object): class BaseOutput(InputOutput): script_class = None - estimator_class = BaseOutputAmountEstimator + estimator_class = BaseOutputEffectiveAmountEstimator - def __init__(self, amount, script): + def __init__(self, amount, script, txid=None): + super(BaseOutput, self).__init__(txid) self.amount = amount # type: int self.script = script # type: BaseOutputScript - self.transaction = None # type: BaseTransaction - self.index = None # type: int - def get_estimator(self, coin): - return self.estimator_class(coin, self) + def get_estimator(self, ledger): + return self.estimator_class(ledger, self) @classmethod def pay_pubkey_hash(cls, amount, pubkey_hash): @@ -145,23 +157,25 @@ class BaseTransaction: input_class = None output_class = None - def __init__(self, raw=None, version=1, locktime=0, height=None, is_saved=False): + def __init__(self, raw=None, version=1, locktime=0): self._raw = raw self._hash = None self._id = None self.version = version # type: int self.locktime = locktime # type: int - self.height = height # type: int self._inputs = [] # type: List[BaseInput] self._outputs = [] # type: List[BaseOutput] - self.is_saved = is_saved # type: bool if raw is not None: self._deserialize() + @property + def hex_id(self): + return hexlify(self.id) + @property def id(self): if self._id is None: - self._id = hexlify(self.hash[::-1]) + self._id = self.hash[::-1] return self._id @property @@ -189,18 +203,19 @@ class BaseTransaction: def outputs(self): # type: () -> ReadOnlyList[BaseOutput] return ReadOnlyList(self._outputs) - def add_inputs(self, inputs): - self._inputs.extend(inputs) + def _add(self, new_ios, existing_ios): + for txio in new_ios: + txio.transaction = self + txio.index = len(existing_ios) + existing_ios.append(txio) self._reset() return self + def add_inputs(self, inputs): + return self._add(inputs, self._inputs) + def add_outputs(self, outputs): - for txo in outputs: - txo.transaction = self - txo.index = len(self._outputs) - self._outputs.append(txo) - self._reset() - return self + return self._add(outputs, self._outputs) @property def fee(self): @@ -260,16 +275,76 @@ class BaseTransaction: ]) self.locktime = stream.read_uint32() - def sign(self, account): # type: (Account) -> BaseTransaction + @classmethod + def get_effective_amount_estimators(cls, funding_accounts): + # type: (Iterable[Account]) -> Generator[BaseOutputEffectiveAmountEstimator] + for account in funding_accounts: + for utxo in account.coin.ledger.get_unspent_outputs(account): + yield utxo.get_estimator(account.coin) + + @classmethod + def ensure_all_have_same_ledger(cls, funding_accounts, change_account=None): + # type: (Iterable[Account], Account) -> baseledger.BaseLedger + ledger = None + for account in funding_accounts: + if ledger is None: + ledger = account.coin.ledger + if ledger != account.coin.ledger: + raise ValueError( + 'All funding accounts used to create a transaction must be on the same ledger.' + ) + if change_account is not None and change_account.coin.ledger != ledger: + raise ValueError('Change account must use same ledger as funding accounts.') + return ledger + + @classmethod + def pay(cls, outputs, funding_accounts, change_account): + """ Efficiently spend utxos from funding_accounts to cover the new outputs. """ + + tx = cls().add_outputs(outputs) + ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account) + amount = ledger.get_transaction_base_fee(tx) + selector = CoinSelector( + list(cls.get_effective_amount_estimators(funding_accounts)), + amount, + ledger.get_input_output_fee( + cls.output_class.pay_pubkey_hash(COIN, NULL_HASH) + ) + ) + + spendables = selector.select() + if not spendables: + raise ValueError('Not enough funds to cover this transaction.') + + spent_sum = sum(s.effective_amount for s in spendables) + if spent_sum > amount: + change_address = change_account.get_least_used_change_address() + change_hash160 = change_account.coin.address_to_hash160(change_address) + change_amount = spent_sum - amount + tx.add_outputs([cls.output_class.pay_pubkey_hash(change_amount, change_hash160)]) + + tx.add_inputs([s.txi for s in spendables]) + tx.sign(funding_accounts) + return tx + + @classmethod + def liquidate(cls, assets, funding_accounts, change_account): + """ Spend assets (utxos) supplementing with funding_accounts if fee is higher than asset value. """ + + def sign(self, funding_accounts): # type: (Iterable[Account]) -> BaseTransaction + ledger = self.ensure_all_have_same_ledger(funding_accounts) for i, txi in enumerate(self._inputs): txo_script = txi.output.script if txo_script.is_pay_pubkey_hash: - address = account.coin.hash160_to_address(txo_script.values['pubkey_hash']) + address = ledger.coin_class.hash160_to_address(txo_script.values['pubkey_hash']) + account = ledger.accounts.get_account_for_address(address) private_key = account.get_private_key_for_address(address) tx = self._serialize_for_signature(i) txi.script.values['signature'] = private_key.sign(tx)+six.int2byte(1) txi.script.values['pubkey'] = private_key.public_key.pubkey_bytes txi.script.generate() + else: + raise NotImplementedError("Don't know how to spend this output.") self._reset() return self diff --git a/torba/coin/bitcoincash.py b/torba/coin/bitcoincash.py new file mode 100644 index 000000000..6f41e126f --- /dev/null +++ b/torba/coin/bitcoincash.py @@ -0,0 +1,85 @@ +__coin__ = 'BitcoinCash' +__node_daemon__ = 'bitcoind' +__node_cli__ = 'bitcoin-cli' +__node_url__ = ( + 'https://download.bitcoinabc.org/0.17.2/linux/bitcoin-abc-0.17.2-x86_64-linux-gnu.tar.gz' +) + +from six import int2byte +from binascii import unhexlify +from torba.baseledger import BaseLedger, BaseHeaders +from torba.basenetwork import BaseNetwork +from torba.basescript import BaseInputScript, BaseOutputScript +from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput +from torba.basecoin import BaseCoin +from torba.basedatabase import BaseSQLiteWalletStorage +from torba.basemanager import BaseWalletManager + + +class WalletManager(BaseWalletManager): + pass + + +class Input(BaseInput): + script_class = BaseInputScript + + +class Output(BaseOutput): + script_class = BaseOutputScript + + +class Transaction(BaseTransaction): + input_class = Input + output_class = Output + + +class BitcoinCashLedger(BaseLedger): + network_class = BaseNetwork + headers_class = BaseHeaders + database_class = BaseSQLiteWalletStorage + + +class MainNetLedger(BitcoinCashLedger): + pass + + +class UnverifiedHeaders(BaseHeaders): + verify_bits_to_target = False + + +class RegTestLedger(BitcoinCashLedger): + headers_class = UnverifiedHeaders + max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206' + genesis_bits = 0x207fffff + target_timespan = 1 + verify_bits_to_target = False + + +class BitcoinCash(BaseCoin): + name = 'BitcoinCash' + symbol = 'BCH' + network = 'mainnet' + + ledger_class = MainNetLedger + transaction_class = Transaction + + pubkey_address_prefix = int2byte(0x00) + script_address_prefix = int2byte(0x05) + extended_public_key_prefix = unhexlify('0488b21e') + extended_private_key_prefix = unhexlify('0488ade4') + + default_fee_per_byte = 50 + + def __init__(self, ledger, fee_per_byte=default_fee_per_byte): + super(BitcoinCash, self).__init__(ledger, fee_per_byte) + + +class BitcoinCashRegtest(BitcoinCash): + network = 'regtest' + ledger_class = RegTestLedger + pubkey_address_prefix = int2byte(111) + script_address_prefix = int2byte(196) + extended_public_key_prefix = unhexlify('043587cf') + extended_private_key_prefix = unhexlify('04358394') + diff --git a/torba/coin/bitcoinsegwit.py b/torba/coin/bitcoinsegwit.py new file mode 100644 index 000000000..82d0fcbb5 --- /dev/null +++ b/torba/coin/bitcoinsegwit.py @@ -0,0 +1,83 @@ +__coin__ = 'BitcoinSegwit' +__node_daemon__ = 'bitcoind' +__node_cli__ = 'bitcoin-cli' +__node_url__ = ( + 'https://bitcoin.org/bin/bitcoin-core-0.16.0/bitcoin-0.16.0-x86_64-linux-gnu.tar.gz' +) + +from six import int2byte +from binascii import unhexlify +from torba.baseledger import BaseLedger, BaseHeaders +from torba.basenetwork import BaseNetwork +from torba.basescript import BaseInputScript, BaseOutputScript +from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput +from torba.basecoin import BaseCoin +from torba.basedatabase import BaseSQLiteWalletStorage +from torba.basemanager import BaseWalletManager + + +class WalletManager(BaseWalletManager): + pass + + +class SQLiteWalletStorage(BaseSQLiteWalletStorage): + pass + + +class Input(BaseInput): + script_class = BaseInputScript + + +class Output(BaseOutput): + script_class = BaseOutputScript + + +class Transaction(BaseTransaction): + input_class = Input + output_class = Output + + +class BitcoinSegwitLedger(BaseLedger): + network_class = BaseNetwork + headers_class = BaseHeaders + + +class MainNetLedger(BitcoinSegwitLedger): + pass + + +class UnverifiedHeaders(BaseHeaders): + verify_bits_to_target = False + + +class RegTestLedger(BitcoinSegwitLedger): + headers_class = UnverifiedHeaders + max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206' + genesis_bits = 0x207fffff + target_timespan = 1 + verify_bits_to_target = False + + +class BitcoinSegwit(BaseCoin): + name = 'BitcoinSegwit' + symbol = 'BTC' + network = 'mainnet' + + ledger_class = MainNetLedger + transaction_class = Transaction + + pubkey_address_prefix = int2byte(0x00) + script_address_prefix = int2byte(0x05) + extended_public_key_prefix = unhexlify('0488b21e') + extended_private_key_prefix = unhexlify('0488ade4') + + default_fee_per_byte = 50 + + def __init__(self, ledger, fee_per_byte=default_fee_per_byte): + super(BitcoinSegwit, self).__init__(ledger, fee_per_byte) + + +class BitcoinSegwitRegtest(BitcoinSegwit): + network = 'regtest' + ledger_class = RegTestLedger diff --git a/torba/coin/btc.py b/torba/coin/btc.py deleted file mode 100644 index d6f23127e..000000000 --- a/torba/coin/btc.py +++ /dev/null @@ -1,43 +0,0 @@ -from six import int2byte -from binascii import unhexlify -from torba.baseledger import BaseLedger -from torba.basenetwork import BaseNetwork -from torba.basescript import BaseInputScript, BaseOutputScript -from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput -from torba.basecoin import BaseCoin - - -class Ledger(BaseLedger): - network_class = BaseNetwork - - -class Input(BaseInput): - script_class = BaseInputScript - - -class Output(BaseOutput): - script_class = BaseOutputScript - - -class Transaction(BaseTransaction): - input_class = Input - output_class = Output - - -class BTC(BaseCoin): - name = 'Bitcoin' - symbol = 'BTC' - network = 'mainnet' - - ledger_class = Ledger - transaction_class = Transaction - - pubkey_address_prefix = int2byte(0x00) - script_address_prefix = int2byte(0x05) - extended_public_key_prefix = unhexlify('0488b21e') - extended_private_key_prefix = unhexlify('0488ade4') - - default_fee_per_byte = 50 - - def __init__(self, ledger, fee_per_byte=default_fee_per_byte): - super(BTC, self).__init__(ledger, fee_per_byte) diff --git a/torba/coinselection.py b/torba/coinselection.py index c0bf38502..398781676 100644 --- a/torba/coinselection.py +++ b/torba/coinselection.py @@ -2,7 +2,7 @@ import six from random import Random from typing import List -from torba.basetransaction import BaseOutputAmountEstimator +import torba MAXIMUM_TRIES = 100000 @@ -10,7 +10,7 @@ MAXIMUM_TRIES = 100000 class CoinSelector: def __init__(self, txos, target, cost_of_change, seed=None): - # type: (List[BaseOutputAmountEstimator], int, int, str) -> None + # type: (List[torba.basetransaction.BaseOutputAmountEstimator], int, int, str) -> None self.txos = txos self.target = target self.cost_of_change = cost_of_change