wip

tests/integration/test_transactions.py

@@ -0,0 +1,32 @@
+import asyncio
+from orchstr8.testcase import IntegrationTestCase
+from torba.constants import COIN
+
+
+class BasicTransactionTests(IntegrationTestCase):
+
+    VERBOSE = True
+
+    async def test_sending_and_recieving(self):
+        account1, account2 = self.account, self.wallet.generate_account(self.ledger)
+
+        self.assertEqual(await self.get_balance(account1), 0)
+        self.assertEqual(await self.get_balance(account2), 0)
+
+        address = await account1.get_least_used_receiving_address().asFuture(asyncio.get_event_loop())
+        sendtxid = await self.blockchain.send_to_address(address.decode(), 5.5)
+        await self.blockchain.generate(1)
+        await self.on_transaction(sendtxid)
+
+        self.assertEqual(await self.get_balance(account1), int(5.5*COIN))
+        self.assertEqual(await self.get_balance(account2), 0)
+
+        address = await account2.get_least_used_receiving_address().asFuture(asyncio.get_event_loop())
+        sendtxid = await self.blockchain.send_to_address(address.decode(), 5.5)
+        await self.broadcast(tx)
+        await self.on_transaction(tx.id.decode())
+        await self.lbrycrd.generate(1)
+
+        self.assertEqual(await self.get_balance(account1), int(3.0*COIN))
+        self.assertEqual(await self.get_balance(account2), int(2.5*COIN))
+

tests/unit/test_account.py

@@ -1,8 +1,8 @@
 from binascii import hexlify
 from twisted.trial import unittest
 
-from torba.coin.btc import BTC
+from torba.coin.bitcoinsegwit import BTC
-from torba.manager import WalletManager
+from torba.basemanager import WalletManager
 from torba.wallet import Account
 
 

tests/unit/test_coinselection.py

@@ -1,9 +1,9 @@
 import unittest
 
-from torba.coin.btc import BTC
+from torba.coin.bitcoinsegwit import BTC
 from torba.coinselection import CoinSelector, MAXIMUM_TRIES
 from torba.constants import CENT
-from torba.manager import WalletManager
+from torba.basemanager import WalletManager
 
 from .test_transaction import Output, get_output as utxo
 

tests/unit/test_transaction.py

@@ -2,9 +2,9 @@ from binascii import hexlify, unhexlify
 from twisted.trial import unittest
 
 from torba.account import Account
-from torba.coin.btc import BTC, Transaction, Output, Input
+from torba.coin.bitcoinsegwit import BTC, Transaction, Output, Input
 from torba.constants import CENT, COIN
-from torba.manager import WalletManager
+from torba.basemanager import WalletManager
 from torba.wallet import Wallet
 
 

tests/unit/test_wallet.py

@@ -1,7 +1,7 @@
 from twisted.trial import unittest
 
-from torba.coin.btc import BTC
+from torba.coin.bitcoinsegwit import BTC
-from torba.manager import WalletManager
+from torba.basemanager import WalletManager
 from torba.wallet import Account, Wallet, WalletStorage
 
 from .ftc import FTC

torba/account.py

@@ -1,6 +1,7 @@
 import itertools
 from typing import Dict, Generator
 from binascii import hexlify, unhexlify
+from twisted.internet import defer
 
 from torba.basecoin import BaseCoin
 from torba.mnemonic import Mnemonic
@@ -20,14 +21,14 @@ class KeyChain:
             for key in child_keys
         ]
 
-    @property
+    @defer.inlineCallbacks
     def has_gap(self):
         if len(self.addresses) < self.minimum_gap:
-            return False
+            defer.returnValue(False)
         for address in self.addresses[-self.minimum_gap:]:
-            if self.coin.ledger.is_address_old(address):
+            if (yield self.coin.ledger.is_address_old(address)):
-                return False
+                defer.returnValue(False)
-        return True
+        defer.returnValue(True)
 
     def generate_next_address(self):
         child_key = self.parent_key.child(len(self.child_keys))
@@ -35,11 +36,12 @@ class KeyChain:
         self.addresses.append(child_key.address)
         return child_key.address
 
+    @defer.inlineCallbacks
     def ensure_enough_addresses(self):
         starting_length = len(self.addresses)
-        while not self.has_gap:
+        while not (yield self.has_gap()):
             self.generate_next_address()
-        return self.addresses[starting_length:]
+        defer.returnValue(self.addresses[starting_length:])
 
 
 class Account:
@@ -135,50 +137,39 @@ class Account:
                 if address == match:
                     return self.private_key.child(a).child(b)
 
+    @defer.inlineCallbacks
     def ensure_enough_addresses(self):
-        return [
+        addresses = []
-            address
+        for keychain in self.keychains:
-            for keychain in self.keychains
+            for address in (yield keychain.ensure_enough_addresses()):
-            for address in keychain.ensure_enough_addresses()
+                addresses.append(address)
-        ]
+        defer.returnValue(addresses)
-
-    def addresses_without_history(self):
-        for address in self.addresses:
-            if not self.coin.ledger.has_address(address):
-                yield address
 
     def get_least_used_receiving_address(self, max_transactions=1000):
         return self._get_least_used_address(
-            self.receiving_keys.addresses,
             self.receiving_keys,
             max_transactions
         )
 
     def get_least_used_change_address(self, max_transactions=100):
         return self._get_least_used_address(
-            self.change_keys.addresses,
             self.change_keys,
             max_transactions
         )
 
-    def _get_least_used_address(self, addresses, keychain, max_transactions):
+    def _get_least_used_address(self, keychain, max_transactions):
         ledger = self.coin.ledger
-        address = ledger.get_least_used_address(addresses, max_transactions)
+        address = ledger.get_least_used_address(self, keychain, max_transactions)
         if address:
             return address
         address = keychain.generate_next_address()
         ledger.subscribe_history(address)
         return address
 
-    def get_unspent_utxos(self):
+    @defer.inlineCallbacks
-        return [
-            utxo
-            for address in self.addresses
-            for utxo in self.coin.ledger.get_unspent_outputs(address)
-        ]
-
     def get_balance(self):
-        return sum(utxo.amount for utxo in self.get_unspent_utxos())
+        utxos = yield self.coin.ledger.get_unspent_outputs(self)
+        defer.returnValue(sum(utxo.amount for utxo in utxos))
 
 
 class AccountsView:
@@ -188,3 +179,13 @@ class AccountsView:
 
     def __iter__(self):  # type: () -> Generator[Account]
         return self._accounts_generator()
+
+    def addresses(self):
+        for account in self:
+            for address in account.addresses:
+                yield address
+
+    def get_account_for_address(self, address):
+        for account in self:
+            if address in account.addresses:
+                return account

torba/basecoin.py

@@ -44,22 +44,13 @@ class BaseCoin(six.with_metaclass(CoinRegistry)):
 
     def __init__(self, ledger, fee_per_byte):
         self.ledger = ledger
-        self.fee_per_byte = fee_per_byte
 
     @classmethod
     def get_id(cls):
         return '{}_{}'.format(cls.symbol.lower(), cls.network.lower())
 
     def to_dict(self):
-        return {'fee_per_byte': self.fee_per_byte}
+        return {}
-
-    def get_input_output_fee(self, io):
-        """ Fee based on size of the input / output. """
-        return self.fee_per_byte * io.size
-
-    def get_transaction_base_fee(self, tx):
-        """ Fee for the transaction header and all outputs; without inputs. """
-        return self.fee_per_byte * tx.base_size
 
     def hash160_to_address(self, h160):
         raw_address = self.pubkey_address_prefix + h160

torba/basedatabase.py

@@ -0,0 +1,219 @@
+import logging
+import os
+import sqlite3
+from twisted.internet import defer
+from twisted.enterprise import adbapi
+
+log = logging.getLogger(__name__)
+
+
+class BaseSQLiteWalletStorage(object):
+
+    CREATE_TX_TABLE = """
+        create table if not exists tx (
+            txid blob primary key,
+            raw blob not null,
+            height integer not null,
+            is_confirmed boolean not null,
+            is_verified boolean not null
+        );
+        create table if not exists address_status (
+            address blob not null,
+            status text not null
+        );
+    """
+
+    CREATE_TXO_TABLE = """
+        create table if not exists txo (
+            txoid integer primary key,
+            account blob not null,
+            address blob not null,
+            txid blob references tx,
+            pos integer not null,
+            amount integer not null,
+            script blob not null
+        );
+    """
+
+    CREATE_TXI_TABLE = """
+        create table if not exists txi (
+            account blob not null,
+            txid blob references tx,
+            txoid integer references txo
+        );
+    """
+
+    CREATE_TABLES_QUERY = (
+        CREATE_TX_TABLE +
+        CREATE_TXO_TABLE +
+        CREATE_TXI_TABLE
+    )
+
+    def __init__(self, ledger):
+        self._db_path = os.path.join(ledger.path, "blockchain.db")
+        self.db = None
+
+    def start(self):
+        log.info("connecting to database: %s", self._db_path)
+        self.db = adbapi.ConnectionPool(
+            'sqlite3', self._db_path, cp_min=1, cp_max=1, check_same_thread=False
+        )
+        return self.db.runInteraction(
+            lambda t: t.executescript(self.CREATE_TABLES_QUERY)
+        )
+
+    def stop(self):
+        self.db.close()
+        return defer.succeed(True)
+
+    @defer.inlineCallbacks
+    def run_and_return_one_or_none(self, query, *args):
+        result = yield self.db.runQuery(query, args)
+        if result:
+            defer.returnValue(result[0][0])
+        else:
+            defer.returnValue(None)
+
+    @defer.inlineCallbacks
+    def run_and_return_list(self, query, *args):
+        result = yield self.db.runQuery(query, args)
+        if result:
+            defer.returnValue([i[0] for i in result])
+        else:
+            defer.returnValue([])
+
+    def run_and_return_id(self, query, *args):
+        def do_save(t):
+            t.execute(query, args)
+            return t.lastrowid
+        return self.db.runInteraction(do_save)
+
+    def add_transaction(self, tx, height, is_confirmed, is_verified):
+        return self.run_and_return_id(
+            "insert into tx values (?, ?, ?, ?, ?)",
+            sqlite3.Binary(tx.id),
+            sqlite3.Binary(tx.raw),
+            height,
+            is_confirmed,
+            is_verified
+        )
+
+    @defer.inlineCallbacks
+    def has_transaction(self, txid):
+        result = yield self.db.runQuery(
+            "select rowid from tx where txid=?", (txid,)
+        )
+        defer.returnValue(bool(result))
+
+    def add_tx_output(self, account, txo):
+        return self.db.runOperation(
+            "insert into txo values (?, ?, ?, ?, ?, ?, ?, ?, ?)", (
+                sqlite3.Binary(account.public_key.address),
+                sqlite3.Binary(txo.script.values['pubkey_hash']),
+                sqlite3.Binary(txo.txid),
+                txo.index,
+                txo.amount,
+                sqlite3.Binary(txo.script.source),
+                txo.script.is_claim_name,
+                txo.script.is_support_claim,
+                txo.script.is_update_claim
+            )
+        )
+
+    def add_tx_input(self, account, txi):
+        def _ops(t):
+            txoid = t.execute(
+                "select rowid from txo where txid=? and pos=?", (
+                    sqlite3.Binary(txi.output_txid), txi.output_index
+                )
+            ).fetchone()[0]
+            t.execute(
+                "insert into txi values (?, ?, ?)", (
+                    sqlite3.Binary(account.public_key.address),
+                    sqlite3.Binary(txi.txid),
+                    txoid
+                )
+            )
+        return self.db.runInteraction(_ops)
+
+    @defer.inlineCallbacks
+    def get_balance_for_account(self, account):
+        result = yield self.db.runQuery(
+            "select sum(amount) from txo where account=:account and rowid not in (select txo from txi where account=:account)",
+            {'account': sqlite3.Binary(account.public_key.address)}
+        )
+        if result:
+            defer.returnValue(result[0][0] or 0)
+        else:
+            defer.returnValue(0)
+
+    def get_used_addresses(self, account):
+        return self.db.runQuery(
+            """
+            SELECT
+              txios.address,
+              sum(txios.used_count) as total
+            FROM
+             (SELECT address, count(*) as used_count FROM txo
+                  WHERE account=:account GROUP BY address
+                UNION
+              SELECT address, count(*) as used_count FROM txi NATURAL JOIN txo
+                  WHERE account=:account GROUP BY address) AS txios
+            GROUP BY txios.address
+            ORDER BY total
+            """, {'account': sqlite3.Binary(account.public_key.address)}
+        )
+
+    @defer.inlineCallbacks
+    def get_earliest_block_height_for_address(self, address):
+        result = yield self.db.runQuery(
+            """
+            SELECT
+              height
+            FROM
+             (SELECT DISTINCT height FROM txi NATURAL JOIN txo NATURAL JOIN tx WHERE address=:address
+                UNION
+              SELECT DISTINCT height FROM txo NATURAL JOIN tx WHERE address=:address) AS txios
+            ORDER BY height LIMIT 1
+            """, {'address': sqlite3.Binary(address)}
+        )
+        if result:
+            defer.returnValue(result[0][0])
+        else:
+            defer.returnValue(None)
+
+    @defer.inlineCallbacks
+    def get_utxos(self, account, output_class):
+        utxos = yield self.db.runQuery(
+            """
+            SELECT
+              amount, script, txid
+            FROM txo
+            WHERE
+              account=:account AND
+              txoid NOT IN (SELECT txoid FROM txi WHERE account=:account)
+            """,
+            {'account': sqlite3.Binary(account.public_key.address)}
+        )
+        defer.returnValue([
+            output_class(
+                values[0],
+                output_class.script_class(values[1]),
+                values[2]
+            ) for values in utxos
+        ])
+
+    @defer.inlineCallbacks
+    def get_address_status(self, address):
+        result = yield self.db.runQuery(
+            "select status from address_status where address=?", (address,)
+        )
+        if result:
+            defer.returnValue(result[0][0])
+        else:
+            defer.returnValue(None)
+
+    def set_address_status(self, address, status):
+        return self.db.runOperation(
+            "replace into address_status (address, status) values (?, ?)", (address,status)
+        )

torba/baseledger.py

@@ -1,17 +1,18 @@
 import os
 import hashlib
+import struct
 from binascii import hexlify, unhexlify
 from typing import List, Dict, Type
 from operator import itemgetter
 
 from twisted.internet import threads, defer, task, reactor
 
+from torba import basetransaction, basedatabase
 from torba.account import Account, AccountsView
 from torba.basecoin import BaseCoin
-from torba.basetransaction import BaseTransaction
 from torba.basenetwork import BaseNetwork
 from torba.stream import StreamController, execute_serially
-from torba.util import hex_to_int, int_to_hex, rev_hex, hash_encode
+from torba.util import int_to_hex, rev_hex, hash_encode
 from torba.hash import double_sha256, pow_hash
 
 
@@ -28,22 +29,11 @@ class Address:
         return len(self.transactions)
 
     def add_transaction(self, transaction):
+        if transaction not in self.transactions:
             self.transactions.append(transaction)
 
-    def get_unspent_utxos(self):
-        inputs, outputs, utxos = [], [], []
-        for tx in self:
-            for txi in tx.inputs:
-                inputs.append((txi.output_txid, txi.output_index))
-            for txo in tx.outputs:
-                if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == self.pubkey_hash:
-                    outputs.append((txo, txo.transaction.hash, txo.index))
-        for output in set(outputs):
-            if output[1:] not in inputs:
-                yield output[0]
 
-
+class BaseLedger(object):
-class BaseLedger:
 
     # coin_class is automatically set by BaseCoin metaclass
     # when it creates the Coin classes, there is a 1..1 relationship
@@ -52,21 +42,38 @@ class BaseLedger:
     # but many coin instances can exist linking back to the single Ledger instance.
     coin_class = None  # type: Type[BaseCoin]
     network_class = None  # type: Type[BaseNetwork]
+    headers_class = None  # type: Type[BaseHeaders]
+    database_class = None  # type: Type[basedatabase.BaseSQLiteWalletStorage]
 
-    verify_bits_to_target = True
+    default_fee_per_byte = 10
 
-    def __init__(self, accounts, config=None, network=None, db=None):
+    def __init__(self, accounts, config=None, db=None, network=None,
+                 fee_per_byte=default_fee_per_byte):
         self.accounts = accounts  # type: AccountsView
         self.config = config or {}
-        self.db = db
+        self.db = db or self.database_class(self)  # type: basedatabase.BaseSQLiteWalletStorage
-        self.addresses = {}  # type: Dict[str, Address]
+        self.network = network or self.network_class(self)
-        self.transactions = {}  # type: Dict[str, BaseTransaction]
-        self.headers = Headers(self)
-        self._on_transaction_controller = StreamController()
-        self.on_transaction = self._on_transaction_controller.stream
-        self.network = network or self.network_class(self.config)
         self.network.on_header.listen(self.process_header)
         self.network.on_status.listen(self.process_status)
+        self.headers = self.headers_class(self)
+        self.fee_per_byte = fee_per_byte
+
+        self._on_transaction_controller = StreamController()
+        self.on_transaction = self._on_transaction_controller.stream
+
+    @property
+    def path(self):
+        return os.path.join(
+            self.config['wallet_path'], self.coin_class.get_id()
+        )
+
+    def get_input_output_fee(self, io):
+        """ Fee based on size of the input / output. """
+        return self.fee_per_byte * io.size
+
+    def get_transaction_base_fee(self, tx):
+        """ Fee for the transaction header and all outputs; without inputs. """
+        return self.fee_per_byte * tx.base_size
 
     @property
     def transaction_class(self):
@@ -77,79 +84,51 @@ class BaseLedger:
         return cls(json_dict)
 
     @defer.inlineCallbacks
-    def load(self):
-        txs = yield self.db.get_transactions()
-        for tx_hash, raw, height in txs:
-            self.transactions[tx_hash] = self.transaction_class(raw, height)
-        txios = yield self.db.get_transaction_inputs_and_outputs()
-        for tx_hash, address_hash, input_output, amount, height in txios:
-            tx = self.transactions[tx_hash]
-            address = self.addresses.get(address_hash)
-            if address is None:
-                address = self.addresses[address_hash] = Address(self.coin_class.address_to_hash160(address_hash))
-            tx.add_txio(address, input_output, amount)
-            address.add_transaction(tx)
-
     def is_address_old(self, address, age_limit=2):
-        age = -1
+        height = yield self.db.get_earliest_block_height_for_address(address)
-        for tx in self.get_transactions(address, []):
+        if height is None:
-            if tx.height == 0:
+            return False
-                tx_age = 0
+        age = self.headers.height - height + 1
-            else:
-                tx_age = self.headers.height - tx.height + 1
-            if tx_age > age:
-                age = tx_age
         return age > age_limit
 
-    def add_transaction(self, address, transaction):  # type: (str, BaseTransaction) -> None
+    @defer.inlineCallbacks
-        if address not in self.addresses:
+    def add_transaction(self, transaction, height):  # type: (basetransaction.BaseTransaction, int) -> None
-            self.addresses[address] = Address(self.coin_class.address_to_hash160(address))
+        yield self.db.add_transaction(transaction, height, False, False)
-        self.addresses[address].add_transaction(transaction)
-        self.transactions.setdefault(transaction.id, transaction)
         self._on_transaction_controller.add(transaction)
 
     def has_address(self, address):
-        return address in self.addresses
+        return address in self.accounts.addresses
 
-    def get_transaction(self, tx_hash, *args):
+    @defer.inlineCallbacks
-        return self.transactions.get(tx_hash, *args)
+    def get_least_used_address(self, account, keychain, max_transactions=100):
+        used_addresses = yield self.db.get_used_addresses(account)
+        unused_set = set(keychain.addresses) - set(map(itemgetter(0), used_addresses))
+        if unused_set:
+            defer.returnValue(unused_set.pop())
+        if used_addresses and used_addresses[0][1] < max_transactions:
+            defer.returnValue(used_addresses[0][0])
 
-    def get_transactions(self, address, *args):
+    def get_unspent_outputs(self, account):
-        return self.addresses.get(address, *args)
+        return self.db.get_utxos(account, self.transaction_class.output_class)
 
-    def get_status(self, address):
+#    def get_unspent_outputs(self, account):
-        hashes = [
+#        inputs, outputs, utxos = set(), set(), set()
-            '{}:{}:'.format(hexlify(tx.hash), tx.height).encode()
+#        for address in self.addresses.values():
-            for tx in self.get_transactions(address, []) if tx.height is not None
+#            for tx in address:
-        ]
+#                for txi in tx.inputs:
-        if hashes:
+#                    inputs.add((hexlify(txi.output_txid), txi.output_index))
-            return hexlify(hashlib.sha256(b''.join(hashes)).digest())
+#                for txo in tx.outputs:
-
+#                    if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == address.pubkey_hash:
-    def has_transaction(self, tx_hash):
+#                        outputs.add((txo, txo.transaction.id, txo.index))
-        return tx_hash in self.transactions
+#        for output in outputs:
-
+#            if output[1:] not in inputs:
-    def get_least_used_address(self, addresses, max_transactions=100):
+#                yield output[0]
-        transaction_counts = []
-        for address in addresses:
-            transactions = self.get_transactions(address, [])
-            tx_count = len(transactions)
-            if tx_count == 0:
-                return address
-            elif tx_count >= max_transactions:
-                continue
-            else:
-                transaction_counts.append((address, tx_count))
-        if transaction_counts:
-            transaction_counts.sort(key=itemgetter(1))
-            return transaction_counts[0]
-
-    def get_unspent_outputs(self, address):
-        if address in self.addresses:
-            return list(self.addresses[address].get_unspent_utxos())
-        return []
 
     @defer.inlineCallbacks
     def start(self):
+        if not os.path.exists(self.path):
+            os.mkdir(self.path)
+        yield self.db.start()
         first_connection = self.network.on_connected.first
         self.network.start()
         yield first_connection
@@ -197,13 +176,14 @@ class BaseLedger:
         # this avoids situation where we're getting status updates to addresses we know
         # need to update anyways. Continue to get history and create more addresses until
         # all missing addresses are created and history for them is fully restored.
-        account.ensure_enough_addresses()
+        yield account.ensure_enough_addresses()
-        addresses = list(account.addresses_without_history())
+        used_addresses = yield self.db.get_used_addresses(account)
+        addresses = set(account.addresses) - set(map(itemgetter(0), used_addresses))
         while addresses:
             yield defer.DeferredList([
                 self.update_history(a) for a in addresses
             ])
-            addresses = account.ensure_enough_addresses()
+            addresses = yield account.ensure_enough_addresses()
 
         # By this point all of the addresses should be restored and we
         # can now subscribe all of them to receive updates.
@@ -212,32 +192,49 @@ class BaseLedger:
             for address in account.addresses
         ])
 
+    def _get_status_from_history(self, history):
+        hashes = [
+            '{}:{}:'.format(hash.decode(), height).encode()
+            for hash, height in map(itemgetter('tx_hash', 'height'), history)
+        ]
+        if hashes:
+            return hexlify(hashlib.sha256(b''.join(hashes)).digest())
+
     @defer.inlineCallbacks
-    def update_history(self, address):
+    def update_history(self, address, remote_status=None):
         history = yield self.network.get_history(address)
-        for hash in map(itemgetter('tx_hash'), history):
+        for hash, height in map(itemgetter('tx_hash', 'height'), history):
-            transaction = self.get_transaction(hash)
+            if not (yield self.db.has_transaction(hash)):
-            if not transaction:
                 raw = yield self.network.get_transaction(hash)
                 transaction = self.transaction_class(unhexlify(raw))
-            self.add_transaction(address, transaction)
+                yield self.add_transaction(transaction, height)
+        if remote_status is None:
+            remote_status = self._get_status_from_history(history)
+        if remote_status:
+            yield self.db.set_address_status(address, remote_status)
 
     @defer.inlineCallbacks
     def subscribe_history(self, address):
-        status = yield self.network.subscribe_address(address)
+        remote_status = yield self.network.subscribe_address(address)
-        if status != self.get_status(address):
+        local_status = yield self.db.get_address_status(address)
-            yield self.update_history(address)
+        if local_status != remote_status:
+            yield self.update_history(address, remote_status)
 
+    @defer.inlineCallbacks
     def process_status(self, response):
-        address, status = response
+        address, remote_status = response
-        if status != self.get_status(address):
+        local_status = yield self.db.get_address_status(address)
-            task.deferLater(reactor, 0, self.update_history, address)
+        if local_status != remote_status:
+            yield self.update_history(address, remote_status)
 
     def broadcast(self, tx):
         return self.network.broadcast(hexlify(tx.raw))
 
 
-class Headers:
+class BaseHeaders:
+
+    header_size = 80
+    verify_bits_to_target = True
 
     def __init__(self, ledger):
         self.ledger = ledger
@@ -247,9 +244,7 @@ class Headers:
 
     @property
     def path(self):
-        wallet_path = self.ledger.config.get('wallet_path', '')
+        return os.path.join(self.ledger.path, 'headers')
-        filename = '{}_headers'.format(self.ledger.coin_class.get_id())
-        return os.path.join(wallet_path, filename)
 
     def touch(self):
         if not os.path.exists(self.path):
@@ -261,13 +256,13 @@ class Headers:
         return len(self) - 1
 
     def sync_read_length(self):
-        return os.path.getsize(self.path) // self.ledger.header_size
+        return os.path.getsize(self.path) // self.header_size
 
     def sync_read_header(self, height):
         if 0 <= height < len(self):
             with open(self.path, 'rb') as f:
-                f.seek(height * self.ledger.header_size)
+                f.seek(height * self.header_size)
-                return f.read(self.ledger.header_size)
+                return f.read(self.header_size)
 
     def __len__(self):
         if self._size is None:
@@ -295,7 +290,7 @@ class Headers:
             previous_header = header
 
         with open(self.path, 'r+b') as f:
-            f.seek(start * self.ledger.header_size)
+            f.seek(start * self.header_size)
             f.write(headers)
             f.truncate()
 
@@ -306,9 +301,9 @@ class Headers:
         self._on_change_controller.add(change)
 
     def _iterate_headers(self, height, headers):
-        assert len(headers) % self.ledger.header_size == 0
+        assert len(headers) % self.header_size == 0
-        for idx in range(len(headers) // self.ledger.header_size):
+        for idx in range(len(headers) // self.header_size):
-            start, end = idx * self.ledger.header_size, (idx + 1) * self.ledger.header_size
+            start, end = idx * self.header_size, (idx + 1) * self.header_size
             header = headers[start:end]
             yield self._deserialize(height+idx, header)
 
@@ -317,15 +312,16 @@ class Headers:
         assert previous_hash == header['prev_block_hash'], \
             "prev hash mismatch: {} vs {}".format(previous_hash, header['prev_block_hash'])
 
-        bits, target = self._calculate_lbry_next_work_required(height, previous_header, header)
+        bits, target = self._calculate_next_work_required(height, previous_header, header)
         assert bits == header['bits'], \
             "bits mismatch: {} vs {} (hash: {})".format(
             bits, header['bits'], self._hash_header(header))
 
-        _pow_hash = self._pow_hash_header(header)
+        # TODO: FIX ME!!!
-        assert int(b'0x' + _pow_hash, 16) <= target, \
+        #_pow_hash = self._pow_hash_header(header)
-            "insufficient proof of work: {} vs target {}".format(
+        #assert int(b'0x' + _pow_hash, 16) <= target, \
-            int(b'0x' + _pow_hash, 16), target)
+        #    "insufficient proof of work: {} vs target {}".format(
+        #    int(b'0x' + _pow_hash, 16), target)
 
     @staticmethod
     def _serialize(header):
@@ -333,7 +329,6 @@ class Headers:
             int_to_hex(header['version'], 4),
             rev_hex(header['prev_block_hash']),
             rev_hex(header['merkle_root']),
-            rev_hex(header['claim_trie_root']),
             int_to_hex(int(header['timestamp']), 4),
             int_to_hex(int(header['bits']), 4),
             int_to_hex(int(header['nonce']), 4)
@@ -341,15 +336,16 @@ class Headers:
 
     @staticmethod
     def _deserialize(height, header):
+        version, = struct.unpack('<I', header[:4])
+        timestamp, bits, nonce = struct.unpack('<III', header[68:80])
         return {
-            'version': hex_to_int(header[0:4]),
+            'block_height': height,
+            'version': version,
             'prev_block_hash': hash_encode(header[4:36]),
             'merkle_root': hash_encode(header[36:68]),
-            'claim_trie_root': hash_encode(header[68:100]),
+            'timestamp': timestamp,
-            'timestamp': hex_to_int(header[100:104]),
+            'bits': bits,
-            'bits': hex_to_int(header[104:108]),
+            'nonce': nonce,
-            'nonce': hex_to_int(header[108:112]),
-            'block_height': height
         }
 
     def _hash_header(self, header):
@@ -362,16 +358,15 @@ class Headers:
             return b'0' * 64
         return hash_encode(pow_hash(unhexlify(self._serialize(header))))
 
-    def _calculate_lbry_next_work_required(self, height, first, last):
+    def _calculate_next_work_required(self, height, first, last):
-        """ See: lbrycrd/src/lbry.cpp """
 
         if height == 0:
             return self.ledger.genesis_bits, self.ledger.max_target
 
-        if self.ledger.verify_bits_to_target:
+        if self.verify_bits_to_target:
             bits = last['bits']
             bitsN = (bits >> 24) & 0xff
-            assert 0x03 <= bitsN <= 0x1f, \
+            assert 0x03 <= bitsN <= 0x1d, \
                 "First part of bits should be in [0x03, 0x1d], but it was {}".format(hex(bitsN))
             bitsBase = bits & 0xffffff
             assert 0x8000 <= bitsBase <= 0x7fffff, \

torba/basemanager.py

@@ -5,13 +5,13 @@ from twisted.internet import defer
 from torba.account import AccountsView
 from torba.basecoin import CoinRegistry
 from torba.baseledger import BaseLedger
-from torba.basetransaction import NULL_HASH
+from torba.basetransaction import BaseTransaction, NULL_HASH
 from torba.coinselection import CoinSelector
 from torba.constants import COIN
 from torba.wallet import Wallet, WalletStorage
 
 
-class WalletManager:
+class BaseWalletManager(object):
 
     def __init__(self, wallets=None, ledgers=None):
         self.wallets = wallets or []  # type: List[Wallet]
@@ -35,10 +35,22 @@ class WalletManager:
         ledger_class = coin_class.ledger_class
         ledger = self.ledgers.get(ledger_class)
         if ledger is None:
-            ledger = ledger_class(self.get_accounts_view(coin_class), ledger_config or {})
+            ledger = self.create_ledger(ledger_class, self.get_accounts_view(coin_class), ledger_config or {})
             self.ledgers[ledger_class] = ledger
         return ledger
 
+    def create_ledger(self, ledger_class, accounts, config):
+        return ledger_class(accounts, config)
+
+    @defer.inlineCallbacks
+    def get_balance(self):
+        balances = {}
+        for ledger in self.ledgers:
+            for account in self.get_accounts(ledger.coin_class):
+                balances.setdefault(ledger.coin_class.name, 0)
+                balances[ledger.coin_class.name] += yield account.get_balance()
+        defer.returnValue(balances)
+
     @property
     def default_wallet(self):
         for wallet in self.wallets:
@@ -72,14 +84,14 @@ class WalletManager:
         return wallet.generate_account(ledger)
 
     @defer.inlineCallbacks
-    def start_ledgers(self):
+    def start(self):
         self.running = True
         yield defer.DeferredList([
             l.start() for l in self.ledgers.values()
         ])
 
     @defer.inlineCallbacks
-    def stop_ledgers(self):
+    def stop(self):
         yield defer.DeferredList([
             l.stop() for l in self.ledgers.values()
         ])
@@ -91,12 +103,13 @@ class WalletManager:
         account = self.default_account
         coin = account.coin
         ledger = coin.ledger
-        tx_class = ledger.transaction_class
+        tx_class = ledger.transaction_class  # type: BaseTransaction
         in_class, out_class = tx_class.input_class, tx_class.output_class
 
         estimators = [
             txo.get_estimator(coin) for txo in account.get_unspent_utxos()
         ]
+        tx_class.create()
 
         cost_of_output = coin.get_input_output_fee(
             out_class.pay_pubkey_hash(COIN, NULL_HASH)

torba/basenetwork.py

@@ -137,8 +137,8 @@ class StratumClientFactory(protocol.ClientFactory):
 
 class BaseNetwork:
 
-    def __init__(self, config):
+    def __init__(self, ledger):
-        self.config = config
+        self.config = ledger.config
         self.client = None
         self.service = None
         self.running = False

torba/basetransaction.py

@@ -1,10 +1,12 @@
 import six
 import logging
-from typing import List
+from typing import List, Iterable, Generator
 from binascii import hexlify
 
-from torba.basecoin import BaseCoin
+from torba import baseledger
 from torba.basescript import BaseInputScript, BaseOutputScript
+from torba.coinselection import CoinSelector
+from torba.constants import COIN
 from torba.bcd_data_stream import BCDataStream
 from torba.hash import sha256
 from torba.account import Account
@@ -19,6 +21,17 @@ NULL_HASH = b'\x00'*32
 
 class InputOutput(object):
 
+    def __init__(self, txid):
+        self._txid = txid  # type: bytes
+        self.transaction = None  # type: BaseTransaction
+        self.index = None  # type: int
+
+    @property
+    def txid(self):
+        if self._txid is None:
+            self._txid = self.transaction.id
+        return self._txid
+
     @property
     def size(self):
         """ Size of this input / output in bytes. """
@@ -37,10 +50,11 @@ class BaseInput(InputOutput):
     NULL_SIGNATURE = b'\x00'*72
     NULL_PUBLIC_KEY = b'\x00'*33
 
-    def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF):
+    def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF, txid=None):
+        super(BaseInput, self).__init__(txid)
         if isinstance(output_or_txid_index, BaseOutput):
             self.output = output_or_txid_index  # type: BaseOutput
-            self.output_txid = self.output.transaction.hash
+            self.output_txid = self.output.transaction.id
             self.output_index = self.output.index
         else:
             self.output = None  # type: BaseOutput
@@ -52,7 +66,7 @@ class BaseInput(InputOutput):
 
     def link_output(self, output):
         assert self.output is None
-        assert self.output_txid == output.transaction.hash
+        assert self.output_txid == output.transaction.id
         assert self.output_index == output.index
         self.output = output
 
@@ -95,15 +109,14 @@ class BaseInput(InputOutput):
         stream.write_uint32(self.sequence)
 
 
-class BaseOutputAmountEstimator(object):
+class BaseOutputEffectiveAmountEstimator(object):
 
     __slots__ = 'coin', 'txi', 'txo', 'fee', 'effective_amount'
 
-    def __init__(self, coin, txo):  # type: (BaseCoin, BaseOutput) -> None
+    def __init__(self, ledger, txo):  # type: (baseledger.BaseLedger, BaseOutput) -> None
-        self.coin = coin
         self.txo = txo
-        self.txi = coin.transaction_class.input_class.spend(txo)
+        self.txi = ledger.transaction_class.input_class.spend(txo)
-        self.fee = coin.get_input_output_fee(self.txi)
+        self.fee = ledger.get_input_output_fee(self.txi)
         self.effective_amount = txo.amount - self.fee
 
     def __lt__(self, other):
@@ -113,16 +126,15 @@ class BaseOutputAmountEstimator(object):
 class BaseOutput(InputOutput):
 
     script_class = None
-    estimator_class = BaseOutputAmountEstimator
+    estimator_class = BaseOutputEffectiveAmountEstimator
 
-    def __init__(self, amount, script):
+    def __init__(self, amount, script, txid=None):
+        super(BaseOutput, self).__init__(txid)
         self.amount = amount  # type: int
         self.script = script  # type: BaseOutputScript
-        self.transaction = None  # type: BaseTransaction
-        self.index = None  # type: int
 
-    def get_estimator(self, coin):
+    def get_estimator(self, ledger):
-        return self.estimator_class(coin, self)
+        return self.estimator_class(ledger, self)
 
     @classmethod
     def pay_pubkey_hash(cls, amount, pubkey_hash):
@@ -145,23 +157,25 @@ class BaseTransaction:
     input_class = None
     output_class = None
 
-    def __init__(self, raw=None, version=1, locktime=0, height=None, is_saved=False):
+    def __init__(self, raw=None, version=1, locktime=0):
         self._raw = raw
         self._hash = None
         self._id = None
         self.version = version  # type: int
         self.locktime = locktime  # type: int
-        self.height = height  # type: int
         self._inputs = []  # type: List[BaseInput]
         self._outputs = []  # type: List[BaseOutput]
-        self.is_saved = is_saved  # type: bool
         if raw is not None:
             self._deserialize()
 
+    @property
+    def hex_id(self):
+        return hexlify(self.id)
+
     @property
     def id(self):
         if self._id is None:
-            self._id = hexlify(self.hash[::-1])
+            self._id = self.hash[::-1]
         return self._id
 
     @property
@@ -189,18 +203,19 @@ class BaseTransaction:
     def outputs(self):  # type: () -> ReadOnlyList[BaseOutput]
         return ReadOnlyList(self._outputs)
 
-    def add_inputs(self, inputs):
+    def _add(self, new_ios, existing_ios):
-        self._inputs.extend(inputs)
+        for txio in new_ios:
+            txio.transaction = self
+            txio.index = len(existing_ios)
+            existing_ios.append(txio)
         self._reset()
         return self
 
+    def add_inputs(self, inputs):
+        return self._add(inputs, self._inputs)
+
     def add_outputs(self, outputs):
-        for txo in outputs:
+        return self._add(outputs, self._outputs)
-            txo.transaction = self
-            txo.index = len(self._outputs)
-            self._outputs.append(txo)
-        self._reset()
-        return self
 
     @property
     def fee(self):
@@ -260,16 +275,76 @@ class BaseTransaction:
             ])
             self.locktime = stream.read_uint32()
 
-    def sign(self, account):  # type: (Account) -> BaseTransaction
+    @classmethod
+    def get_effective_amount_estimators(cls, funding_accounts):
+        # type: (Iterable[Account]) -> Generator[BaseOutputEffectiveAmountEstimator]
+        for account in funding_accounts:
+            for utxo in account.coin.ledger.get_unspent_outputs(account):
+                yield utxo.get_estimator(account.coin)
+
+    @classmethod
+    def ensure_all_have_same_ledger(cls, funding_accounts, change_account=None):
+        # type: (Iterable[Account], Account) -> baseledger.BaseLedger
+        ledger = None
+        for account in funding_accounts:
+            if ledger is None:
+                ledger = account.coin.ledger
+            if ledger != account.coin.ledger:
+                raise ValueError(
+                    'All funding accounts used to create a transaction must be on the same ledger.'
+                )
+        if change_account is not None and change_account.coin.ledger != ledger:
+            raise ValueError('Change account must use same ledger as funding accounts.')
+        return ledger
+
+    @classmethod
+    def pay(cls, outputs, funding_accounts, change_account):
+        """ Efficiently spend utxos from funding_accounts to cover the new outputs. """
+
+        tx = cls().add_outputs(outputs)
+        ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
+        amount = ledger.get_transaction_base_fee(tx)
+        selector = CoinSelector(
+            list(cls.get_effective_amount_estimators(funding_accounts)),
+            amount,
+            ledger.get_input_output_fee(
+                cls.output_class.pay_pubkey_hash(COIN, NULL_HASH)
+            )
+        )
+
+        spendables = selector.select()
+        if not spendables:
+            raise ValueError('Not enough funds to cover this transaction.')
+
+        spent_sum = sum(s.effective_amount for s in spendables)
+        if spent_sum > amount:
+            change_address = change_account.get_least_used_change_address()
+            change_hash160 = change_account.coin.address_to_hash160(change_address)
+            change_amount = spent_sum - amount
+            tx.add_outputs([cls.output_class.pay_pubkey_hash(change_amount, change_hash160)])
+
+        tx.add_inputs([s.txi for s in spendables])
+        tx.sign(funding_accounts)
+        return tx
+
+    @classmethod
+    def liquidate(cls, assets, funding_accounts, change_account):
+        """ Spend assets (utxos) supplementing with funding_accounts if fee is higher than asset value. """
+
+    def sign(self, funding_accounts):  # type: (Iterable[Account]) -> BaseTransaction
+        ledger = self.ensure_all_have_same_ledger(funding_accounts)
         for i, txi in enumerate(self._inputs):
             txo_script = txi.output.script
             if txo_script.is_pay_pubkey_hash:
-                address = account.coin.hash160_to_address(txo_script.values['pubkey_hash'])
+                address = ledger.coin_class.hash160_to_address(txo_script.values['pubkey_hash'])
+                account = ledger.accounts.get_account_for_address(address)
                 private_key = account.get_private_key_for_address(address)
                 tx = self._serialize_for_signature(i)
                 txi.script.values['signature'] = private_key.sign(tx)+six.int2byte(1)
                 txi.script.values['pubkey'] = private_key.public_key.pubkey_bytes
                 txi.script.generate()
+            else:
+                raise NotImplementedError("Don't know how to spend this output.")
         self._reset()
         return self
 

torba/coin/bitcoincash.py

@@ -0,0 +1,85 @@
+__coin__ = 'BitcoinCash'
+__node_daemon__ = 'bitcoind'
+__node_cli__ = 'bitcoin-cli'
+__node_url__ = (
+    'https://download.bitcoinabc.org/0.17.2/linux/bitcoin-abc-0.17.2-x86_64-linux-gnu.tar.gz'
+)
+
+from six import int2byte
+from binascii import unhexlify
+from torba.baseledger import BaseLedger, BaseHeaders
+from torba.basenetwork import BaseNetwork
+from torba.basescript import BaseInputScript, BaseOutputScript
+from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput
+from torba.basecoin import BaseCoin
+from torba.basedatabase import BaseSQLiteWalletStorage
+from torba.basemanager import BaseWalletManager
+
+
+class WalletManager(BaseWalletManager):
+    pass
+
+
+class Input(BaseInput):
+    script_class = BaseInputScript
+
+
+class Output(BaseOutput):
+    script_class = BaseOutputScript
+
+
+class Transaction(BaseTransaction):
+    input_class = Input
+    output_class = Output
+
+
+class BitcoinCashLedger(BaseLedger):
+    network_class = BaseNetwork
+    headers_class = BaseHeaders
+    database_class = BaseSQLiteWalletStorage
+
+
+class MainNetLedger(BitcoinCashLedger):
+    pass
+
+
+class UnverifiedHeaders(BaseHeaders):
+    verify_bits_to_target = False
+
+
+class RegTestLedger(BitcoinCashLedger):
+    headers_class = UnverifiedHeaders
+    max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+    genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206'
+    genesis_bits = 0x207fffff
+    target_timespan = 1
+    verify_bits_to_target = False
+
+
+class BitcoinCash(BaseCoin):
+    name = 'BitcoinCash'
+    symbol = 'BCH'
+    network = 'mainnet'
+
+    ledger_class = MainNetLedger
+    transaction_class = Transaction
+
+    pubkey_address_prefix = int2byte(0x00)
+    script_address_prefix = int2byte(0x05)
+    extended_public_key_prefix = unhexlify('0488b21e')
+    extended_private_key_prefix = unhexlify('0488ade4')
+
+    default_fee_per_byte = 50
+
+    def __init__(self, ledger, fee_per_byte=default_fee_per_byte):
+        super(BitcoinCash, self).__init__(ledger, fee_per_byte)
+
+
+class BitcoinCashRegtest(BitcoinCash):
+    network = 'regtest'
+    ledger_class = RegTestLedger
+    pubkey_address_prefix = int2byte(111)
+    script_address_prefix = int2byte(196)
+    extended_public_key_prefix = unhexlify('043587cf')
+    extended_private_key_prefix = unhexlify('04358394')
+

torba/coin/bitcoinsegwit.py

@@ -0,0 +1,83 @@
+__coin__ = 'BitcoinSegwit'
+__node_daemon__ = 'bitcoind'
+__node_cli__ = 'bitcoin-cli'
+__node_url__ = (
+    'https://bitcoin.org/bin/bitcoin-core-0.16.0/bitcoin-0.16.0-x86_64-linux-gnu.tar.gz'
+)
+
+from six import int2byte
+from binascii import unhexlify
+from torba.baseledger import BaseLedger, BaseHeaders
+from torba.basenetwork import BaseNetwork
+from torba.basescript import BaseInputScript, BaseOutputScript
+from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput
+from torba.basecoin import BaseCoin
+from torba.basedatabase import BaseSQLiteWalletStorage
+from torba.basemanager import BaseWalletManager
+
+
+class WalletManager(BaseWalletManager):
+    pass
+
+
+class SQLiteWalletStorage(BaseSQLiteWalletStorage):
+    pass
+
+
+class Input(BaseInput):
+    script_class = BaseInputScript
+
+
+class Output(BaseOutput):
+    script_class = BaseOutputScript
+
+
+class Transaction(BaseTransaction):
+    input_class = Input
+    output_class = Output
+
+
+class BitcoinSegwitLedger(BaseLedger):
+    network_class = BaseNetwork
+    headers_class = BaseHeaders
+
+
+class MainNetLedger(BitcoinSegwitLedger):
+    pass
+
+
+class UnverifiedHeaders(BaseHeaders):
+    verify_bits_to_target = False
+
+
+class RegTestLedger(BitcoinSegwitLedger):
+    headers_class = UnverifiedHeaders
+    max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+    genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206'
+    genesis_bits = 0x207fffff
+    target_timespan = 1
+    verify_bits_to_target = False
+
+
+class BitcoinSegwit(BaseCoin):
+    name = 'BitcoinSegwit'
+    symbol = 'BTC'
+    network = 'mainnet'
+
+    ledger_class = MainNetLedger
+    transaction_class = Transaction
+
+    pubkey_address_prefix = int2byte(0x00)
+    script_address_prefix = int2byte(0x05)
+    extended_public_key_prefix = unhexlify('0488b21e')
+    extended_private_key_prefix = unhexlify('0488ade4')
+
+    default_fee_per_byte = 50
+
+    def __init__(self, ledger, fee_per_byte=default_fee_per_byte):
+        super(BitcoinSegwit, self).__init__(ledger, fee_per_byte)
+
+
+class BitcoinSegwitRegtest(BitcoinSegwit):
+    network = 'regtest'
+    ledger_class = RegTestLedger

torba/coin/btc.py

@@ -1,43 +0,0 @@
-from six import int2byte
-from binascii import unhexlify
-from torba.baseledger import BaseLedger
-from torba.basenetwork import BaseNetwork
-from torba.basescript import BaseInputScript, BaseOutputScript
-from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput
-from torba.basecoin import BaseCoin
-
-
-class Ledger(BaseLedger):
-    network_class = BaseNetwork
-
-
-class Input(BaseInput):
-    script_class = BaseInputScript
-
-
-class Output(BaseOutput):
-    script_class = BaseOutputScript
-
-
-class Transaction(BaseTransaction):
-    input_class = Input
-    output_class = Output
-
-
-class BTC(BaseCoin):
-    name = 'Bitcoin'
-    symbol = 'BTC'
-    network = 'mainnet'
-
-    ledger_class = Ledger
-    transaction_class = Transaction
-
-    pubkey_address_prefix = int2byte(0x00)
-    script_address_prefix = int2byte(0x05)
-    extended_public_key_prefix = unhexlify('0488b21e')
-    extended_private_key_prefix = unhexlify('0488ade4')
-
-    default_fee_per_byte = 50
-
-    def __init__(self, ledger, fee_per_byte=default_fee_per_byte):
-        super(BTC, self).__init__(ledger, fee_per_byte)

torba/coinselection.py

@@ -2,7 +2,7 @@ import six
 from random import Random
 from typing import List
 
-from torba.basetransaction import BaseOutputAmountEstimator
+import torba
 
 MAXIMUM_TRIES = 100000
 
@@ -10,7 +10,7 @@ MAXIMUM_TRIES = 100000
 class CoinSelector:
 
     def __init__(self, txos, target, cost_of_change, seed=None):
-        # type: (List[BaseOutputAmountEstimator], int, int, str) -> None
+        # type: (List[torba.basetransaction.BaseOutputAmountEstimator], int, int, str) -> None
         self.txos = txos
         self.target = target
         self.cost_of_change = cost_of_change