LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

wip

Browse source
This commit is contained in:
Lex Berezhny 2018-06-07 23:47:46 -04:00
parent 54350bcbc4
commit 78e4135159
17 changed files with 708 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
tests/integration/__init__.py Normal file
View file

32
tests/integration/test_transactions.py Normal file
View file

@ -0,0 +1,32 @@
import asyncio
from orchstr8.testcase import IntegrationTestCase
from torba.constants import COIN
class BasicTransactionTests(IntegrationTestCase):
VERBOSE = True
async def test_sending_and_recieving(self):
account1, account2 = self.account, self.wallet.generate_account(self.ledger)
self.assertEqual(await self.get_balance(account1), 0)
self.assertEqual(await self.get_balance(account2), 0)
address = await account1.get_least_used_receiving_address().asFuture(asyncio.get_event_loop())
sendtxid = await self.blockchain.send_to_address(address.decode(), 5.5)
await self.blockchain.generate(1)
await self.on_transaction(sendtxid)
self.assertEqual(await self.get_balance(account1), int(5.5*COIN))
self.assertEqual(await self.get_balance(account2), 0)
address = await account2.get_least_used_receiving_address().asFuture(asyncio.get_event_loop())
sendtxid = await self.blockchain.send_to_address(address.decode(), 5.5)
await self.broadcast(tx)
await self.on_transaction(tx.id.decode())
await self.lbrycrd.generate(1)
self.assertEqual(await self.get_balance(account1), int(3.0*COIN))
self.assertEqual(await self.get_balance(account2), int(2.5*COIN))

4
tests/unit/test_account.py
View file

@ -1,8 +1,8 @@
from binascii import hexlify
from twisted.trial import unittest
from torba.coin.btc import BTC
from torba.manager import WalletManager
from torba.coin.bitcoinsegwit import BTC
from torba.basemanager import WalletManager
from torba.wallet import Account

4
tests/unit/test_coinselection.py
View file

@ -1,9 +1,9 @@
import unittest
from torba.coin.btc import BTC
from torba.coin.bitcoinsegwit import BTC
from torba.coinselection import CoinSelector, MAXIMUM_TRIES
from torba.constants import CENT
from torba.manager import WalletManager
from torba.basemanager import WalletManager
from .test_transaction import Output, get_output as utxo

4
tests/unit/test_transaction.py
View file

@ -2,9 +2,9 @@ from binascii import hexlify, unhexlify
from twisted.trial import unittest
from torba.account import Account
from torba.coin.btc import BTC, Transaction, Output, Input
from torba.coin.bitcoinsegwit import BTC, Transaction, Output, Input
from torba.constants import CENT, COIN
from torba.manager import WalletManager
from torba.basemanager import WalletManager
from torba.wallet import Wallet

4
tests/unit/test_wallet.py
View file

@ -1,7 +1,7 @@
from twisted.trial import unittest
from torba.coin.btc import BTC
from torba.manager import WalletManager
from torba.coin.bitcoinsegwit import BTC
from torba.basemanager import WalletManager
from torba.wallet import Account, Wallet, WalletStorage
from .ftc import FTC

59
torba/account.py
View file

@ -1,6 +1,7 @@
import itertools
from typing import Dict, Generator
from binascii import hexlify, unhexlify
from twisted.internet import defer
from torba.basecoin import BaseCoin
from torba.mnemonic import Mnemonic
@ -20,14 +21,14 @@ class KeyChain:
for key in child_keys
]
@property
@defer.inlineCallbacks
def has_gap(self):
if len(self.addresses) < self.minimum_gap:
return False
defer.returnValue(False)
for address in self.addresses[-self.minimum_gap:]:
if self.coin.ledger.is_address_old(address):
return False
return True
if (yield self.coin.ledger.is_address_old(address)):
defer.returnValue(False)
defer.returnValue(True)
def generate_next_address(self):
child_key = self.parent_key.child(len(self.child_keys))
@ -35,11 +36,12 @@ class KeyChain:
self.addresses.append(child_key.address)
return child_key.address
@defer.inlineCallbacks
def ensure_enough_addresses(self):
starting_length = len(self.addresses)
while not self.has_gap:
while not (yield self.has_gap()):
self.generate_next_address()
return self.addresses[starting_length:]
defer.returnValue(self.addresses[starting_length:])
class Account:
@ -135,50 +137,39 @@ class Account:
if address == match:
return self.private_key.child(a).child(b)
@defer.inlineCallbacks
def ensure_enough_addresses(self):
return [
address
for keychain in self.keychains
for address in keychain.ensure_enough_addresses()
]
def addresses_without_history(self):
for address in self.addresses:
if not self.coin.ledger.has_address(address):
yield address
addresses = []
for keychain in self.keychains:
for address in (yield keychain.ensure_enough_addresses()):
addresses.append(address)
defer.returnValue(addresses)
def get_least_used_receiving_address(self, max_transactions=1000):
return self._get_least_used_address(
self.receiving_keys.addresses,
self.receiving_keys,
max_transactions
)
def get_least_used_change_address(self, max_transactions=100):
return self._get_least_used_address(
self.change_keys.addresses,
self.change_keys,
max_transactions
)
def _get_least_used_address(self, addresses, keychain, max_transactions):
def _get_least_used_address(self, keychain, max_transactions):
ledger = self.coin.ledger
address = ledger.get_least_used_address(addresses, max_transactions)
address = ledger.get_least_used_address(self, keychain, max_transactions)
if address:
return address
address = keychain.generate_next_address()
ledger.subscribe_history(address)
return address
def get_unspent_utxos(self):
return [
utxo
for address in self.addresses
for utxo in self.coin.ledger.get_unspent_outputs(address)
]
@defer.inlineCallbacks
def get_balance(self):
return sum(utxo.amount for utxo in self.get_unspent_utxos())
utxos = yield self.coin.ledger.get_unspent_outputs(self)
defer.returnValue(sum(utxo.amount for utxo in utxos))
class AccountsView:
@ -188,3 +179,13 @@ class AccountsView:
def __iter__(self): # type: () -> Generator[Account]
return self._accounts_generator()
def addresses(self):
for account in self:
for address in account.addresses:
yield address
def get_account_for_address(self, address):
for account in self:
if address in account.addresses:
return account

11
torba/basecoin.py
View file

@ -44,22 +44,13 @@ class BaseCoin(six.with_metaclass(CoinRegistry)):
def __init__(self, ledger, fee_per_byte):
self.ledger = ledger
self.fee_per_byte = fee_per_byte
@classmethod
def get_id(cls):
return '{}_{}'.format(cls.symbol.lower(), cls.network.lower())
def to_dict(self):
return {'fee_per_byte': self.fee_per_byte}
def get_input_output_fee(self, io):
""" Fee based on size of the input / output. """
return self.fee_per_byte * io.size
def get_transaction_base_fee(self, tx):
""" Fee for the transaction header and all outputs; without inputs. """
return self.fee_per_byte * tx.base_size
return {}
def hash160_to_address(self, h160):
raw_address = self.pubkey_address_prefix + h160

219
torba/basedatabase.py Normal file
View file

@ -0,0 +1,219 @@
import logging
import os
import sqlite3
from twisted.internet import defer
from twisted.enterprise import adbapi
log = logging.getLogger(__name__)
class BaseSQLiteWalletStorage(object):
CREATE_TX_TABLE = """
create table if not exists tx (
txid blob primary key,
raw blob not null,
height integer not null,
is_confirmed boolean not null,
is_verified boolean not null
);
create table if not exists address_status (
address blob not null,
status text not null
);
"""
CREATE_TXO_TABLE = """
create table if not exists txo (
txoid integer primary key,
account blob not null,
address blob not null,
txid blob references tx,
pos integer not null,
amount integer not null,
script blob not null
);
"""
CREATE_TXI_TABLE = """
create table if not exists txi (
account blob not null,
txid blob references tx,
txoid integer references txo
);
"""
CREATE_TABLES_QUERY = (
CREATE_TX_TABLE +
CREATE_TXO_TABLE +
CREATE_TXI_TABLE
)
def __init__(self, ledger):
self._db_path = os.path.join(ledger.path, "blockchain.db")
self.db = None
def start(self):
log.info("connecting to database: %s", self._db_path)
self.db = adbapi.ConnectionPool(
'sqlite3', self._db_path, cp_min=1, cp_max=1, check_same_thread=False
)
return self.db.runInteraction(
lambda t: t.executescript(self.CREATE_TABLES_QUERY)
)
def stop(self):
self.db.close()
return defer.succeed(True)
@defer.inlineCallbacks
def run_and_return_one_or_none(self, query, *args):
result = yield self.db.runQuery(query, args)
if result:
defer.returnValue(result[0][0])
else:
defer.returnValue(None)
@defer.inlineCallbacks
def run_and_return_list(self, query, *args):
result = yield self.db.runQuery(query, args)
if result:
defer.returnValue([i[0] for i in result])
else:
defer.returnValue([])
def run_and_return_id(self, query, *args):
def do_save(t):
t.execute(query, args)
return t.lastrowid
return self.db.runInteraction(do_save)
def add_transaction(self, tx, height, is_confirmed, is_verified):
return self.run_and_return_id(
"insert into tx values (?, ?, ?, ?, ?)",
sqlite3.Binary(tx.id),
sqlite3.Binary(tx.raw),
height,
is_confirmed,
is_verified
)
@defer.inlineCallbacks
def has_transaction(self, txid):
result = yield self.db.runQuery(
"select rowid from tx where txid=?", (txid,)
)
defer.returnValue(bool(result))
def add_tx_output(self, account, txo):
return self.db.runOperation(
"insert into txo values (?, ?, ?, ?, ?, ?, ?, ?, ?)", (
sqlite3.Binary(account.public_key.address),
sqlite3.Binary(txo.script.values['pubkey_hash']),
sqlite3.Binary(txo.txid),
txo.index,
txo.amount,
sqlite3.Binary(txo.script.source),
txo.script.is_claim_name,
txo.script.is_support_claim,
txo.script.is_update_claim
)
)
def add_tx_input(self, account, txi):
def _ops(t):
txoid = t.execute(
"select rowid from txo where txid=? and pos=?", (
sqlite3.Binary(txi.output_txid), txi.output_index
)
).fetchone()[0]
t.execute(
"insert into txi values (?, ?, ?)", (
sqlite3.Binary(account.public_key.address),
sqlite3.Binary(txi.txid),
txoid
)
)
return self.db.runInteraction(_ops)
@defer.inlineCallbacks
def get_balance_for_account(self, account):
result = yield self.db.runQuery(
"select sum(amount) from txo where account=:account and rowid not in (select txo from txi where account=:account)",
{'account': sqlite3.Binary(account.public_key.address)}
)
if result:
defer.returnValue(result[0][0] or 0)
else:
defer.returnValue(0)
def get_used_addresses(self, account):
return self.db.runQuery(
"""
SELECT
txios.address,
sum(txios.used_count) as total
FROM
(SELECT address, count(*) as used_count FROM txo
WHERE account=:account GROUP BY address
UNION
SELECT address, count(*) as used_count FROM txi NATURAL JOIN txo
WHERE account=:account GROUP BY address) AS txios
GROUP BY txios.address
ORDER BY total
""", {'account': sqlite3.Binary(account.public_key.address)}
)
@defer.inlineCallbacks
def get_earliest_block_height_for_address(self, address):
result = yield self.db.runQuery(
"""
SELECT
height
FROM
(SELECT DISTINCT height FROM txi NATURAL JOIN txo NATURAL JOIN tx WHERE address=:address
UNION
SELECT DISTINCT height FROM txo NATURAL JOIN tx WHERE address=:address) AS txios
ORDER BY height LIMIT 1
""", {'address': sqlite3.Binary(address)}
)
if result:
defer.returnValue(result[0][0])
else:
defer.returnValue(None)
@defer.inlineCallbacks
def get_utxos(self, account, output_class):
utxos = yield self.db.runQuery(
"""
SELECT
amount, script, txid
FROM txo
WHERE
account=:account AND
txoid NOT IN (SELECT txoid FROM txi WHERE account=:account)
""",
{'account': sqlite3.Binary(account.public_key.address)}
)
defer.returnValue([
output_class(
values[0],
output_class.script_class(values[1]),
values[2]
) for values in utxos
])
@defer.inlineCallbacks
def get_address_status(self, address):
result = yield self.db.runQuery(
"select status from address_status where address=?", (address,)
)
if result:
defer.returnValue(result[0][0])
else:
defer.returnValue(None)
def set_address_status(self, address, status):
return self.db.runOperation(
"replace into address_status (address, status) values (?, ?)", (address,status)
)

247
torba/baseledger.py
View file

@ -1,17 +1,18 @@
import os
import hashlib
import struct
from binascii import hexlify, unhexlify
from typing import List, Dict, Type
from operator import itemgetter
from twisted.internet import threads, defer, task, reactor
from torba import basetransaction, basedatabase
from torba.account import Account, AccountsView
from torba.basecoin import BaseCoin
from torba.basetransaction import BaseTransaction
from torba.basenetwork import BaseNetwork
from torba.stream import StreamController, execute_serially
from torba.util import hex_to_int, int_to_hex, rev_hex, hash_encode
from torba.util import int_to_hex, rev_hex, hash_encode
from torba.hash import double_sha256, pow_hash
@ -28,22 +29,11 @@ class Address:
return len(self.transactions)
def add_transaction(self, transaction):
if transaction not in self.transactions:
self.transactions.append(transaction)
def get_unspent_utxos(self):
inputs, outputs, utxos = [], [], []
for tx in self:
for txi in tx.inputs:
inputs.append((txi.output_txid, txi.output_index))
for txo in tx.outputs:
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == self.pubkey_hash:
outputs.append((txo, txo.transaction.hash, txo.index))
for output in set(outputs):
if output[1:] not in inputs:
yield output[0]
class BaseLedger:
class BaseLedger(object):
# coin_class is automatically set by BaseCoin metaclass
# when it creates the Coin classes, there is a 1..1 relationship
@ -52,21 +42,38 @@ class BaseLedger:
# but many coin instances can exist linking back to the single Ledger instance.
coin_class = None # type: Type[BaseCoin]
network_class = None # type: Type[BaseNetwork]
headers_class = None # type: Type[BaseHeaders]
database_class = None # type: Type[basedatabase.BaseSQLiteWalletStorage]
verify_bits_to_target = True
default_fee_per_byte = 10
def __init__(self, accounts, config=None, network=None, db=None):
def __init__(self, accounts, config=None, db=None, network=None,
fee_per_byte=default_fee_per_byte):
self.accounts = accounts # type: AccountsView
self.config = config or {}
self.db = db
self.addresses = {} # type: Dict[str, Address]
self.transactions = {} # type: Dict[str, BaseTransaction]
self.headers = Headers(self)
self._on_transaction_controller = StreamController()
self.on_transaction = self._on_transaction_controller.stream
self.network = network or self.network_class(self.config)
self.db = db or self.database_class(self) # type: basedatabase.BaseSQLiteWalletStorage
self.network = network or self.network_class(self)
self.network.on_header.listen(self.process_header)
self.network.on_status.listen(self.process_status)
self.headers = self.headers_class(self)
self.fee_per_byte = fee_per_byte
self._on_transaction_controller = StreamController()
self.on_transaction = self._on_transaction_controller.stream
@property
def path(self):
return os.path.join(
self.config['wallet_path'], self.coin_class.get_id()
)
def get_input_output_fee(self, io):
""" Fee based on size of the input / output. """
return self.fee_per_byte * io.size
def get_transaction_base_fee(self, tx):
""" Fee for the transaction header and all outputs; without inputs. """
return self.fee_per_byte * tx.base_size
@property
def transaction_class(self):
@ -77,79 +84,51 @@ class BaseLedger:
return cls(json_dict)
@defer.inlineCallbacks
def load(self):
txs = yield self.db.get_transactions()
for tx_hash, raw, height in txs:
self.transactions[tx_hash] = self.transaction_class(raw, height)
txios = yield self.db.get_transaction_inputs_and_outputs()
for tx_hash, address_hash, input_output, amount, height in txios:
tx = self.transactions[tx_hash]
address = self.addresses.get(address_hash)
if address is None:
address = self.addresses[address_hash] = Address(self.coin_class.address_to_hash160(address_hash))
tx.add_txio(address, input_output, amount)
address.add_transaction(tx)
def is_address_old(self, address, age_limit=2):
age = -1
for tx in self.get_transactions(address, []):
if tx.height == 0:
tx_age = 0
else:
tx_age = self.headers.height - tx.height + 1
if tx_age > age:
age = tx_age
height = yield self.db.get_earliest_block_height_for_address(address)
if height is None:
return False
age = self.headers.height - height + 1
return age > age_limit
def add_transaction(self, address, transaction): # type: (str, BaseTransaction) -> None
if address not in self.addresses:
self.addresses[address] = Address(self.coin_class.address_to_hash160(address))
self.addresses[address].add_transaction(transaction)
self.transactions.setdefault(transaction.id, transaction)
@defer.inlineCallbacks
def add_transaction(self, transaction, height): # type: (basetransaction.BaseTransaction, int) -> None
yield self.db.add_transaction(transaction, height, False, False)
self._on_transaction_controller.add(transaction)
def has_address(self, address):
return address in self.addresses
return address in self.accounts.addresses
def get_transaction(self, tx_hash, *args):
return self.transactions.get(tx_hash, *args)
@defer.inlineCallbacks
def get_least_used_address(self, account, keychain, max_transactions=100):
used_addresses = yield self.db.get_used_addresses(account)
unused_set = set(keychain.addresses) - set(map(itemgetter(0), used_addresses))
if unused_set:
defer.returnValue(unused_set.pop())
if used_addresses and used_addresses[0][1] < max_transactions:
defer.returnValue(used_addresses[0][0])
def get_transactions(self, address, *args):
return self.addresses.get(address, *args)
def get_unspent_outputs(self, account):
return self.db.get_utxos(account, self.transaction_class.output_class)
def get_status(self, address):
hashes = [
'{}:{}:'.format(hexlify(tx.hash), tx.height).encode()
for tx in self.get_transactions(address, []) if tx.height is not None
]
if hashes:
return hexlify(hashlib.sha256(b''.join(hashes)).digest())
def has_transaction(self, tx_hash):
return tx_hash in self.transactions
def get_least_used_address(self, addresses, max_transactions=100):
transaction_counts = []
for address in addresses:
transactions = self.get_transactions(address, [])
tx_count = len(transactions)
if tx_count == 0:
return address
elif tx_count >= max_transactions:
continue
else:
transaction_counts.append((address, tx_count))
if transaction_counts:
transaction_counts.sort(key=itemgetter(1))
return transaction_counts[0]
def get_unspent_outputs(self, address):
if address in self.addresses:
return list(self.addresses[address].get_unspent_utxos())
return []
# def get_unspent_outputs(self, account):
# inputs, outputs, utxos = set(), set(), set()
# for address in self.addresses.values():
# for tx in address:
# for txi in tx.inputs:
# inputs.add((hexlify(txi.output_txid), txi.output_index))
# for txo in tx.outputs:
# if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == address.pubkey_hash:
# outputs.add((txo, txo.transaction.id, txo.index))
# for output in outputs:
# if output[1:] not in inputs:
# yield output[0]
@defer.inlineCallbacks
def start(self):
if not os.path.exists(self.path):
os.mkdir(self.path)
yield self.db.start()
first_connection = self.network.on_connected.first
self.network.start()
yield first_connection
@ -197,13 +176,14 @@ class BaseLedger:
# this avoids situation where we're getting status updates to addresses we know
# need to update anyways. Continue to get history and create more addresses until
# all missing addresses are created and history for them is fully restored.
account.ensure_enough_addresses()
addresses = list(account.addresses_without_history())
yield account.ensure_enough_addresses()
used_addresses = yield self.db.get_used_addresses(account)
addresses = set(account.addresses) - set(map(itemgetter(0), used_addresses))
while addresses:
yield defer.DeferredList([
self.update_history(a) for a in addresses
])
addresses = account.ensure_enough_addresses()
addresses = yield account.ensure_enough_addresses()
# By this point all of the addresses should be restored and we
# can now subscribe all of them to receive updates.
@ -212,32 +192,49 @@ class BaseLedger:
for address in account.addresses
])
def _get_status_from_history(self, history):
hashes = [
'{}:{}:'.format(hash.decode(), height).encode()
for hash, height in map(itemgetter('tx_hash', 'height'), history)
]
if hashes:
return hexlify(hashlib.sha256(b''.join(hashes)).digest())
@defer.inlineCallbacks
def update_history(self, address):
def update_history(self, address, remote_status=None):
history = yield self.network.get_history(address)
for hash in map(itemgetter('tx_hash'), history):
transaction = self.get_transaction(hash)
if not transaction:
for hash, height in map(itemgetter('tx_hash', 'height'), history):
if not (yield self.db.has_transaction(hash)):
raw = yield self.network.get_transaction(hash)
transaction = self.transaction_class(unhexlify(raw))
self.add_transaction(address, transaction)
yield self.add_transaction(transaction, height)
if remote_status is None:
remote_status = self._get_status_from_history(history)
if remote_status:
yield self.db.set_address_status(address, remote_status)
@defer.inlineCallbacks
def subscribe_history(self, address):
status = yield self.network.subscribe_address(address)
if status != self.get_status(address):
yield self.update_history(address)
remote_status = yield self.network.subscribe_address(address)
local_status = yield self.db.get_address_status(address)
if local_status != remote_status:
yield self.update_history(address, remote_status)
@defer.inlineCallbacks
def process_status(self, response):
address, status = response
if status != self.get_status(address):
task.deferLater(reactor, 0, self.update_history, address)
address, remote_status = response
local_status = yield self.db.get_address_status(address)
if local_status != remote_status:
yield self.update_history(address, remote_status)
def broadcast(self, tx):
return self.network.broadcast(hexlify(tx.raw))
class Headers:
class BaseHeaders:
header_size = 80
verify_bits_to_target = True
def __init__(self, ledger):
self.ledger = ledger
@ -247,9 +244,7 @@ class Headers:
@property
def path(self):
wallet_path = self.ledger.config.get('wallet_path', '')
filename = '{}_headers'.format(self.ledger.coin_class.get_id())
return os.path.join(wallet_path, filename)
return os.path.join(self.ledger.path, 'headers')
def touch(self):
if not os.path.exists(self.path):
@ -261,13 +256,13 @@ class Headers:
return len(self) - 1
def sync_read_length(self):
return os.path.getsize(self.path) // self.ledger.header_size
return os.path.getsize(self.path) // self.header_size
def sync_read_header(self, height):
if 0 <= height < len(self):
with open(self.path, 'rb') as f:
f.seek(height * self.ledger.header_size)
return f.read(self.ledger.header_size)
f.seek(height * self.header_size)
return f.read(self.header_size)
def __len__(self):
if self._size is None:
@ -295,7 +290,7 @@ class Headers:
previous_header = header
with open(self.path, 'r+b') as f:
f.seek(start * self.ledger.header_size)
f.seek(start * self.header_size)
f.write(headers)
f.truncate()
@ -306,9 +301,9 @@ class Headers:
self._on_change_controller.add(change)
def _iterate_headers(self, height, headers):
assert len(headers) % self.ledger.header_size == 0
for idx in range(len(headers) // self.ledger.header_size):
start, end = idx * self.ledger.header_size, (idx + 1) * self.ledger.header_size
assert len(headers) % self.header_size == 0
for idx in range(len(headers) // self.header_size):
start, end = idx * self.header_size, (idx + 1) * self.header_size
header = headers[start:end]
yield self._deserialize(height+idx, header)
@ -317,15 +312,16 @@ class Headers:
assert previous_hash == header['prev_block_hash'], \
"prev hash mismatch: {} vs {}".format(previous_hash, header['prev_block_hash'])
bits, target = self._calculate_lbry_next_work_required(height, previous_header, header)
bits, target = self._calculate_next_work_required(height, previous_header, header)
assert bits == header['bits'], \
"bits mismatch: {} vs {} (hash: {})".format(
bits, header['bits'], self._hash_header(header))
_pow_hash = self._pow_hash_header(header)
assert int(b'0x' + _pow_hash, 16) <= target, \
"insufficient proof of work: {} vs target {}".format(
int(b'0x' + _pow_hash, 16), target)
# TODO: FIX ME!!!
#_pow_hash = self._pow_hash_header(header)
#assert int(b'0x' + _pow_hash, 16) <= target, \
# "insufficient proof of work: {} vs target {}".format(
# int(b'0x' + _pow_hash, 16), target)
@staticmethod
def _serialize(header):
@ -333,7 +329,6 @@ class Headers:
int_to_hex(header['version'], 4),
rev_hex(header['prev_block_hash']),
rev_hex(header['merkle_root']),
rev_hex(header['claim_trie_root']),
int_to_hex(int(header['timestamp']), 4),
int_to_hex(int(header['bits']), 4),
int_to_hex(int(header['nonce']), 4)
@ -341,15 +336,16 @@ class Headers:
@staticmethod
def _deserialize(height, header):
version, = struct.unpack('<I', header[:4])
timestamp, bits, nonce = struct.unpack('<III', header[68:80])
return {
'version': hex_to_int(header[0:4]),
'block_height': height,
'version': version,
'prev_block_hash': hash_encode(header[4:36]),
'merkle_root': hash_encode(header[36:68]),
'claim_trie_root': hash_encode(header[68:100]),
'timestamp': hex_to_int(header[100:104]),
'bits': hex_to_int(header[104:108]),
'nonce': hex_to_int(header[108:112]),
'block_height': height
'timestamp': timestamp,
'bits': bits,
'nonce': nonce,
}
def _hash_header(self, header):
@ -362,16 +358,15 @@ class Headers:
return b'0' * 64
return hash_encode(pow_hash(unhexlify(self._serialize(header))))
def _calculate_lbry_next_work_required(self, height, first, last):
""" See: lbrycrd/src/lbry.cpp """
def _calculate_next_work_required(self, height, first, last):
if height == 0:
return self.ledger.genesis_bits, self.ledger.max_target
if self.ledger.verify_bits_to_target:
if self.verify_bits_to_target:
bits = last['bits']
bitsN = (bits >> 24) & 0xff
assert 0x03 <= bitsN <= 0x1f, \
assert 0x03 <= bitsN <= 0x1d, \
"First part of bits should be in [0x03, 0x1d], but it was {}".format(hex(bitsN))
bitsBase = bits & 0xffffff
assert 0x8000 <= bitsBase <= 0x7fffff, \

25
torba/manager.py → torba/basemanager.py
View file

@ -5,13 +5,13 @@ from twisted.internet import defer
from torba.account import AccountsView
from torba.basecoin import CoinRegistry
from torba.baseledger import BaseLedger
from torba.basetransaction import NULL_HASH
from torba.basetransaction import BaseTransaction, NULL_HASH
from torba.coinselection import CoinSelector
from torba.constants import COIN
from torba.wallet import Wallet, WalletStorage
class WalletManager:
class BaseWalletManager(object):
def __init__(self, wallets=None, ledgers=None):
self.wallets = wallets or [] # type: List[Wallet]
@ -35,10 +35,22 @@ class WalletManager:
ledger_class = coin_class.ledger_class
ledger = self.ledgers.get(ledger_class)
if ledger is None:
ledger = ledger_class(self.get_accounts_view(coin_class), ledger_config or {})
ledger = self.create_ledger(ledger_class, self.get_accounts_view(coin_class), ledger_config or {})
self.ledgers[ledger_class] = ledger
return ledger
def create_ledger(self, ledger_class, accounts, config):
return ledger_class(accounts, config)
@defer.inlineCallbacks
def get_balance(self):
balances = {}
for ledger in self.ledgers:
for account in self.get_accounts(ledger.coin_class):
balances.setdefault(ledger.coin_class.name, 0)
balances[ledger.coin_class.name] += yield account.get_balance()
defer.returnValue(balances)
@property
def default_wallet(self):
for wallet in self.wallets:
@ -72,14 +84,14 @@ class WalletManager:
return wallet.generate_account(ledger)
@defer.inlineCallbacks
def start_ledgers(self):
def start(self):
self.running = True
yield defer.DeferredList([
l.start() for l in self.ledgers.values()
])
@defer.inlineCallbacks
def stop_ledgers(self):
def stop(self):
yield defer.DeferredList([
l.stop() for l in self.ledgers.values()
])
@ -91,12 +103,13 @@ class WalletManager:
account = self.default_account
coin = account.coin
ledger = coin.ledger
tx_class = ledger.transaction_class
tx_class = ledger.transaction_class # type: BaseTransaction
in_class, out_class = tx_class.input_class, tx_class.output_class
estimators = [
txo.get_estimator(coin) for txo in account.get_unspent_utxos()
]
tx_class.create()
cost_of_output = coin.get_input_output_fee(
out_class.pay_pubkey_hash(COIN, NULL_HASH)

4
torba/basenetwork.py
View file

@ -137,8 +137,8 @@ class StratumClientFactory(protocol.ClientFactory):
class BaseNetwork:
def __init__(self, config):
self.config = config
def __init__(self, ledger):
self.config = ledger.config
self.client = None
self.service = None
self.running = False

135
torba/basetransaction.py
View file

@ -1,10 +1,12 @@
import six
import logging
from typing import List
from typing import List, Iterable, Generator
from binascii import hexlify
from torba.basecoin import BaseCoin
from torba import baseledger
from torba.basescript import BaseInputScript, BaseOutputScript
from torba.coinselection import CoinSelector
from torba.constants import COIN
from torba.bcd_data_stream import BCDataStream
from torba.hash import sha256
from torba.account import Account
@ -19,6 +21,17 @@ NULL_HASH = b'\x00'*32
class InputOutput(object):
def __init__(self, txid):
self._txid = txid # type: bytes
self.transaction = None # type: BaseTransaction
self.index = None # type: int
@property
def txid(self):
if self._txid is None:
self._txid = self.transaction.id
return self._txid
@property
def size(self):
""" Size of this input / output in bytes. """
@ -37,10 +50,11 @@ class BaseInput(InputOutput):
NULL_SIGNATURE = b'\x00'*72
NULL_PUBLIC_KEY = b'\x00'*33
def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF):
def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF, txid=None):
super(BaseInput, self).__init__(txid)
if isinstance(output_or_txid_index, BaseOutput):
self.output = output_or_txid_index # type: BaseOutput
self.output_txid = self.output.transaction.hash
self.output_txid = self.output.transaction.id
self.output_index = self.output.index
else:
self.output = None # type: BaseOutput
@ -52,7 +66,7 @@ class BaseInput(InputOutput):
def link_output(self, output):
assert self.output is None
assert self.output_txid == output.transaction.hash
assert self.output_txid == output.transaction.id
assert self.output_index == output.index
self.output = output
@ -95,15 +109,14 @@ class BaseInput(InputOutput):
stream.write_uint32(self.sequence)
class BaseOutputAmountEstimator(object):
class BaseOutputEffectiveAmountEstimator(object):
__slots__ = 'coin', 'txi', 'txo', 'fee', 'effective_amount'
def __init__(self, coin, txo): # type: (BaseCoin, BaseOutput) -> None
self.coin = coin
def __init__(self, ledger, txo): # type: (baseledger.BaseLedger, BaseOutput) -> None
self.txo = txo
self.txi = coin.transaction_class.input_class.spend(txo)
self.fee = coin.get_input_output_fee(self.txi)
self.txi = ledger.transaction_class.input_class.spend(txo)
self.fee = ledger.get_input_output_fee(self.txi)
self.effective_amount = txo.amount - self.fee
def __lt__(self, other):
@ -113,16 +126,15 @@ class BaseOutputAmountEstimator(object):
class BaseOutput(InputOutput):
script_class = None
estimator_class = BaseOutputAmountEstimator
estimator_class = BaseOutputEffectiveAmountEstimator
def __init__(self, amount, script):
def __init__(self, amount, script, txid=None):
super(BaseOutput, self).__init__(txid)
self.amount = amount # type: int
self.script = script # type: BaseOutputScript
self.transaction = None # type: BaseTransaction
self.index = None # type: int
def get_estimator(self, coin):
return self.estimator_class(coin, self)
def get_estimator(self, ledger):
return self.estimator_class(ledger, self)
@classmethod
def pay_pubkey_hash(cls, amount, pubkey_hash):
@ -145,23 +157,25 @@ class BaseTransaction:
input_class = None
output_class = None
def __init__(self, raw=None, version=1, locktime=0, height=None, is_saved=False):
def __init__(self, raw=None, version=1, locktime=0):
self._raw = raw
self._hash = None
self._id = None
self.version = version # type: int
self.locktime = locktime # type: int
self.height = height # type: int
self._inputs = [] # type: List[BaseInput]
self._outputs = [] # type: List[BaseOutput]
self.is_saved = is_saved # type: bool
if raw is not None:
self._deserialize()
@property
def hex_id(self):
return hexlify(self.id)
@property
def id(self):
if self._id is None:
self._id = hexlify(self.hash[::-1])
self._id = self.hash[::-1]
return self._id
@property
@ -189,18 +203,19 @@ class BaseTransaction:
def outputs(self): # type: () -> ReadOnlyList[BaseOutput]
return ReadOnlyList(self._outputs)
def add_inputs(self, inputs):
self._inputs.extend(inputs)
def _add(self, new_ios, existing_ios):
for txio in new_ios:
txio.transaction = self
txio.index = len(existing_ios)
existing_ios.append(txio)
self._reset()
return self
def add_inputs(self, inputs):
return self._add(inputs, self._inputs)
def add_outputs(self, outputs):
for txo in outputs:
txo.transaction = self
txo.index = len(self._outputs)
self._outputs.append(txo)
self._reset()
return self
return self._add(outputs, self._outputs)
@property
def fee(self):
@ -260,16 +275,76 @@ class BaseTransaction:
])
self.locktime = stream.read_uint32()
def sign(self, account): # type: (Account) -> BaseTransaction
@classmethod
def get_effective_amount_estimators(cls, funding_accounts):
# type: (Iterable[Account]) -> Generator[BaseOutputEffectiveAmountEstimator]
for account in funding_accounts:
for utxo in account.coin.ledger.get_unspent_outputs(account):
yield utxo.get_estimator(account.coin)
@classmethod
def ensure_all_have_same_ledger(cls, funding_accounts, change_account=None):
# type: (Iterable[Account], Account) -> baseledger.BaseLedger
ledger = None
for account in funding_accounts:
if ledger is None:
ledger = account.coin.ledger
if ledger != account.coin.ledger:
raise ValueError(
'All funding accounts used to create a transaction must be on the same ledger.'
)
if change_account is not None and change_account.coin.ledger != ledger:
raise ValueError('Change account must use same ledger as funding accounts.')
return ledger
@classmethod
def pay(cls, outputs, funding_accounts, change_account):
""" Efficiently spend utxos from funding_accounts to cover the new outputs. """
tx = cls().add_outputs(outputs)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
amount = ledger.get_transaction_base_fee(tx)
selector = CoinSelector(
list(cls.get_effective_amount_estimators(funding_accounts)),
amount,
ledger.get_input_output_fee(
cls.output_class.pay_pubkey_hash(COIN, NULL_HASH)
)
)
spendables = selector.select()
if not spendables:
raise ValueError('Not enough funds to cover this transaction.')
spent_sum = sum(s.effective_amount for s in spendables)
if spent_sum > amount:
change_address = change_account.get_least_used_change_address()
change_hash160 = change_account.coin.address_to_hash160(change_address)
change_amount = spent_sum - amount
tx.add_outputs([cls.output_class.pay_pubkey_hash(change_amount, change_hash160)])
tx.add_inputs([s.txi for s in spendables])
tx.sign(funding_accounts)
return tx
@classmethod
def liquidate(cls, assets, funding_accounts, change_account):
""" Spend assets (utxos) supplementing with funding_accounts if fee is higher than asset value. """
def sign(self, funding_accounts): # type: (Iterable[Account]) -> BaseTransaction
ledger = self.ensure_all_have_same_ledger(funding_accounts)
for i, txi in enumerate(self._inputs):
txo_script = txi.output.script
if txo_script.is_pay_pubkey_hash:
address = account.coin.hash160_to_address(txo_script.values['pubkey_hash'])
address = ledger.coin_class.hash160_to_address(txo_script.values['pubkey_hash'])
account = ledger.accounts.get_account_for_address(address)
private_key = account.get_private_key_for_address(address)
tx = self._serialize_for_signature(i)
txi.script.values['signature'] = private_key.sign(tx)+six.int2byte(1)
txi.script.values['pubkey'] = private_key.public_key.pubkey_bytes
txi.script.generate()
else:
raise NotImplementedError("Don't know how to spend this output.")
self._reset()
return self

85
torba/coin/bitcoincash.py Normal file
View file

@ -0,0 +1,85 @@
__coin__ = 'BitcoinCash'
__node_daemon__ = 'bitcoind'
__node_cli__ = 'bitcoin-cli'
__node_url__ = (
'https://download.bitcoinabc.org/0.17.2/linux/bitcoin-abc-0.17.2-x86_64-linux-gnu.tar.gz'
)
from six import int2byte
from binascii import unhexlify
from torba.baseledger import BaseLedger, BaseHeaders
from torba.basenetwork import BaseNetwork
from torba.basescript import BaseInputScript, BaseOutputScript
from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput
from torba.basecoin import BaseCoin
from torba.basedatabase import BaseSQLiteWalletStorage
from torba.basemanager import BaseWalletManager
class WalletManager(BaseWalletManager):
pass
class Input(BaseInput):
script_class = BaseInputScript
class Output(BaseOutput):
script_class = BaseOutputScript
class Transaction(BaseTransaction):
input_class = Input
output_class = Output
class BitcoinCashLedger(BaseLedger):
network_class = BaseNetwork
headers_class = BaseHeaders
database_class = BaseSQLiteWalletStorage
class MainNetLedger(BitcoinCashLedger):
pass
class UnverifiedHeaders(BaseHeaders):
verify_bits_to_target = False
class RegTestLedger(BitcoinCashLedger):
headers_class = UnverifiedHeaders
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206'
genesis_bits = 0x207fffff
target_timespan = 1
verify_bits_to_target = False
class BitcoinCash(BaseCoin):
name = 'BitcoinCash'
symbol = 'BCH'
network = 'mainnet'
ledger_class = MainNetLedger
transaction_class = Transaction
pubkey_address_prefix = int2byte(0x00)
script_address_prefix = int2byte(0x05)
extended_public_key_prefix = unhexlify('0488b21e')
extended_private_key_prefix = unhexlify('0488ade4')
default_fee_per_byte = 50
def __init__(self, ledger, fee_per_byte=default_fee_per_byte):
super(BitcoinCash, self).__init__(ledger, fee_per_byte)
class BitcoinCashRegtest(BitcoinCash):
network = 'regtest'
ledger_class = RegTestLedger
pubkey_address_prefix = int2byte(111)
script_address_prefix = int2byte(196)
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')

83
torba/coin/bitcoinsegwit.py Normal file
View file

@ -0,0 +1,83 @@
__coin__ = 'BitcoinSegwit'
__node_daemon__ = 'bitcoind'
__node_cli__ = 'bitcoin-cli'
__node_url__ = (
'https://bitcoin.org/bin/bitcoin-core-0.16.0/bitcoin-0.16.0-x86_64-linux-gnu.tar.gz'
)
from six import int2byte
from binascii import unhexlify
from torba.baseledger import BaseLedger, BaseHeaders
from torba.basenetwork import BaseNetwork
from torba.basescript import BaseInputScript, BaseOutputScript
from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput
from torba.basecoin import BaseCoin
from torba.basedatabase import BaseSQLiteWalletStorage
from torba.basemanager import BaseWalletManager
class WalletManager(BaseWalletManager):
pass
class SQLiteWalletStorage(BaseSQLiteWalletStorage):
pass
class Input(BaseInput):
script_class = BaseInputScript
class Output(BaseOutput):
script_class = BaseOutputScript
class Transaction(BaseTransaction):
input_class = Input
output_class = Output
class BitcoinSegwitLedger(BaseLedger):
network_class = BaseNetwork
headers_class = BaseHeaders
class MainNetLedger(BitcoinSegwitLedger):
pass
class UnverifiedHeaders(BaseHeaders):
verify_bits_to_target = False
class RegTestLedger(BitcoinSegwitLedger):
headers_class = UnverifiedHeaders
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206'
genesis_bits = 0x207fffff
target_timespan = 1
verify_bits_to_target = False
class BitcoinSegwit(BaseCoin):
name = 'BitcoinSegwit'
symbol = 'BTC'
network = 'mainnet'
ledger_class = MainNetLedger
transaction_class = Transaction
pubkey_address_prefix = int2byte(0x00)
script_address_prefix = int2byte(0x05)
extended_public_key_prefix = unhexlify('0488b21e')
extended_private_key_prefix = unhexlify('0488ade4')
default_fee_per_byte = 50
def __init__(self, ledger, fee_per_byte=default_fee_per_byte):
super(BitcoinSegwit, self).__init__(ledger, fee_per_byte)
class BitcoinSegwitRegtest(BitcoinSegwit):
network = 'regtest'
ledger_class = RegTestLedger

43
torba/coin/btc.py
View file

@ -1,43 +0,0 @@
from six import int2byte
from binascii import unhexlify
from torba.baseledger import BaseLedger
from torba.basenetwork import BaseNetwork
from torba.basescript import BaseInputScript, BaseOutputScript
from torba.basetransaction import BaseTransaction, BaseInput, BaseOutput
from torba.basecoin import BaseCoin
class Ledger(BaseLedger):
network_class = BaseNetwork
class Input(BaseInput):
script_class = BaseInputScript
class Output(BaseOutput):
script_class = BaseOutputScript
class Transaction(BaseTransaction):
input_class = Input
output_class = Output
class BTC(BaseCoin):
name = 'Bitcoin'
symbol = 'BTC'
network = 'mainnet'
ledger_class = Ledger
transaction_class = Transaction
pubkey_address_prefix = int2byte(0x00)
script_address_prefix = int2byte(0x05)
extended_public_key_prefix = unhexlify('0488b21e')
extended_private_key_prefix = unhexlify('0488ade4')
default_fee_per_byte = 50
def __init__(self, ledger, fee_per_byte=default_fee_per_byte):
super(BTC, self).__init__(ledger, fee_per_byte)

4
torba/coinselection.py
View file

@ -2,7 +2,7 @@ import six
from random import Random
from typing import List
from torba.basetransaction import BaseOutputAmountEstimator
import torba
MAXIMUM_TRIES = 100000
@ -10,7 +10,7 @@ MAXIMUM_TRIES = 100000
class CoinSelector:
def __init__(self, txos, target, cost_of_change, seed=None):
# type: (List[BaseOutputAmountEstimator], int, int, str) -> None
# type: (List[torba.basetransaction.BaseOutputAmountEstimator], int, int, str) -> None
self.txos = txos
self.target = target
self.cost_of_change = cost_of_change