LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

switched from blob to text for most columns, added a new TXRef/TXORef referencing scheme

Browse source
This commit is contained in:
Lex Berezhny 2018-07-14 21:34:07 -04:00
parent 9370b1d2fa
commit 184d3a5910
15 changed files with 281 additions and 225 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
tests/integration/test_transactions.py
View file

@ -14,9 +14,9 @@ class BasicTransactionTests(IntegrationTestCase):
self.assertEqual(await self.get_balance(account2), 0)
sendtxids = []
for i in range(9):
for i in range(5):
address1 = await d2f(account1.receiving.get_or_create_usable_address())
sendtxid = await self.blockchain.send_to_address(address1.decode(), 1.1)
sendtxid = await self.blockchain.send_to_address(address1, 1.1)
sendtxids.append(sendtxid)
await self.on_transaction_id(sendtxid) # mempool
await self.blockchain.generate(1)
@ -24,7 +24,7 @@ class BasicTransactionTests(IntegrationTestCase):
self.on_transaction_id(txid) for txid in sendtxids
])
self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 9.9)
self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 5.5)
self.assertEqual(round(await self.get_balance(account2)/COIN, 1), 0)
address2 = await d2f(account2.receiving.get_or_create_usable_address())
@ -38,13 +38,9 @@ class BasicTransactionTests(IntegrationTestCase):
await self.blockchain.generate(1)
await self.on_transaction(tx) # confirmed
self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 7.9)
self.assertEqual(round(await self.get_balance(account1)/COIN, 1), 3.5)
self.assertEqual(round(await self.get_balance(account2)/COIN, 1), 2.0)
all_balances = await d2f(self.manager.get_balance())
self.assertIn(self.ledger.get_id(), all_balances)
self.assertEqual(round(all_balances[self.ledger.get_id()]/COIN, 1), 9.9)
utxos = await d2f(self.account.get_unspent_outputs())
tx = await d2f(self.ledger.transaction_class.liquidate(
[utxos[0]], [account1], account1

43
tests/unit/test_account.py
View file

@ -94,31 +94,30 @@ class TestKeyChainAccount(unittest.TestCase):
def test_generate_account_from_seed(self):
account = self.ledger.account_class.from_seed(
self.ledger,
u"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab"
u"sent",
u"torba", receiving_gap=3, change_gap=2
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab"
"sent", "torba", receiving_gap=3, change_gap=2
)
self.assertEqual(
account.private_key.extended_key_string(),
b'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P'
b'6yz3jMbycrLrRMpeAJxR8qDg8'
'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P'
'6yz3jMbycrLrRMpeAJxR8qDg8'
)
self.assertEqual(
account.public_key.extended_key_string(),
b'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f'
b'iW44g14WF52fYC5J483wqQ5ZP'
'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f'
'iW44g14WF52fYC5J483wqQ5ZP'
)
address = yield account.receiving.ensure_address_gap()
self.assertEqual(address[0], b'1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP')
self.assertEqual(address[0], '1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP')
private_key = yield self.ledger.get_private_key_for_address(b'1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP')
private_key = yield self.ledger.get_private_key_for_address('1PmX9T3sCiDysNtWszJa44SkKcpGc2NaXP')
self.assertEqual(
private_key.extended_key_string(),
b'xprv9xNEfQ296VTRaEUDZ8oKq74xw2U6kpj486vFUB4K1wT9U25GX4UwuzFgJN1YuRrqkQ5TTwCpkYnjNpSoH'
b'SBaEigNHPkoeYbuPMRo6mRUjxg'
'xprv9xNEfQ296VTRaEUDZ8oKq74xw2U6kpj486vFUB4K1wT9U25GX4UwuzFgJN1YuRrqkQ5TTwCpkYnjNpSoH'
'SBaEigNHPkoeYbuPMRo6mRUjxg'
)
invalid_key = yield self.ledger.get_private_key_for_address(b'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
invalid_key = yield self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
self.assertIsNone(invalid_key)
self.assertEqual(
@ -247,20 +246,18 @@ class TestSingleKeyAccount(unittest.TestCase):
def test_generate_account_from_seed(self):
account = self.ledger.account_class.from_seed(
self.ledger,
u"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab"
u"sent",
u"torba",
is_hd=False
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab"
"sent", "torba", is_hd=False
)
self.assertEqual(
account.private_key.extended_key_string(),
b'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P'
b'6yz3jMbycrLrRMpeAJxR8qDg8'
'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P'
'6yz3jMbycrLrRMpeAJxR8qDg8'
)
self.assertEqual(
account.public_key.extended_key_string(),
b'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f'
b'iW44g14WF52fYC5J483wqQ5ZP'
'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f'
'iW44g14WF52fYC5J483wqQ5ZP'
)
address = yield account.receiving.ensure_address_gap()
self.assertEqual(address[0], account.public_key.address)
@ -268,11 +265,11 @@ class TestSingleKeyAccount(unittest.TestCase):
private_key = yield self.ledger.get_private_key_for_address(address[0])
self.assertEqual(
private_key.extended_key_string(),
b'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P'
b'6yz3jMbycrLrRMpeAJxR8qDg8'
'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P'
'6yz3jMbycrLrRMpeAJxR8qDg8'
)
invalid_key = yield self.ledger.get_private_key_for_address(b'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
invalid_key = yield self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
self.assertIsNone(invalid_key)
self.assertEqual(

2
tests/unit/test_bip32.py
View file

@ -51,7 +51,7 @@ class BIP32Tests(unittest.TestCase):
self.assertEqual(
ec_point.y(), 86198965946979720220333266272536217633917099472454294641561154971209433250106
)
self.assertEqual(private_key.address(), b'1GVM5dEhThbiyCZ9gqBZBv6p9whga7MTXo' )
self.assertEqual(private_key.address(), '1GVM5dEhThbiyCZ9gqBZBv6p9whga7MTXo' )
with self.assertRaisesRegex(ValueError, 'invalid BIP32 private key child number'):
private_key.child(-1)
self.assertIsInstance(private_key.child(PrivateKey.HARDENED), PrivateKey)

14
tests/unit/test_ledger.py
View file

@ -30,7 +30,7 @@ class MockNetwork:
def get_transaction(self, tx_hash):
self.get_transaction_called.append(tx_hash)
return defer.succeed(self.transaction[tx_hash.decode()])
return defer.succeed(self.transaction[tx_hash])
class MockHeaders:
@ -70,9 +70,9 @@ class TestSynchronization(unittest.TestCase):
self.ledger.headers.height = 3
self.ledger.network = MockNetwork([
{'tx_hash': b'abcd01', 'height': 1},
{'tx_hash': b'abcd02', 'height': 2},
{'tx_hash': b'abcd03', 'height': 3},
{'tx_hash': 'abcd01', 'height': 1},
{'tx_hash': 'abcd02', 'height': 2},
{'tx_hash': 'abcd03', 'height': 3},
], {
'abcd01': hexlify(get_transaction(get_output(1)).raw),
'abcd02': hexlify(get_transaction(get_output(2)).raw),
@ -80,7 +80,7 @@ class TestSynchronization(unittest.TestCase):
})
yield self.ledger.update_history(address)
self.assertEqual(self.ledger.network.get_history_called, [address])
self.assertEqual(self.ledger.network.get_transaction_called, [b'abcd01', b'abcd02', b'abcd03'])
self.assertEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03'])
address_details = yield self.ledger.db.get_address(address)
self.assertEqual(address_details['history'], 'abcd01:1:abcd02:2:abcd03:3:')
@ -91,12 +91,12 @@ class TestSynchronization(unittest.TestCase):
self.assertEqual(self.ledger.network.get_history_called, [address])
self.assertEqual(self.ledger.network.get_transaction_called, [])
self.ledger.network.history.append({'tx_hash': b'abcd04', 'height': 4})
self.ledger.network.history.append({'tx_hash': 'abcd04', 'height': 4})
self.ledger.network.transaction['abcd04'] = hexlify(get_transaction(get_output(4)).raw)
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
yield self.ledger.update_history(address)
self.assertEqual(self.ledger.network.get_history_called, [address])
self.assertEqual(self.ledger.network.get_transaction_called, [b'abcd04'])
self.assertEqual(self.ledger.network.get_transaction_called, ['abcd04'])
address_details = yield self.ledger.db.get_address(address)
self.assertEqual(address_details['history'], 'abcd01:1:abcd02:2:abcd03:3:abcd04:4:')

42
tests/unit/test_transaction.py
View file

@ -71,11 +71,11 @@ class TestTransactionSerialization(unittest.TestCase):
self.assertEqual(len(tx.outputs), 1)
coinbase = tx.inputs[0]
self.assertEqual(coinbase.output_txhash, NULL_HASH)
self.assertEqual(coinbase.output_index, 0xFFFFFFFF)
self.assertTrue(coinbase.txo_ref.is_null, NULL_HASH)
self.assertEqual(coinbase.txo_ref.position, 0xFFFFFFFF)
self.assertEqual(coinbase.sequence, 4294967295)
self.assertTrue(coinbase.is_coinbase)
self.assertEqual(coinbase.script, None)
self.assertIsNotNone(coinbase.coinbase)
self.assertIsNone(coinbase.script)
self.assertEqual(
coinbase.coinbase[8:],
b'The Times 03/Jan/2009 Chancellor on brink of second bailout for banks'
@ -83,7 +83,7 @@ class TestTransactionSerialization(unittest.TestCase):
out = tx.outputs[0]
self.assertEqual(out.amount, 5000000000)
self.assertEqual(out.index, 0)
self.assertEqual(out.position, 0)
self.assertTrue(out.script.is_pay_pubkey)
self.assertFalse(out.script.is_pay_pubkey_hash)
self.assertFalse(out.script.is_pay_script_hash)
@ -106,19 +106,16 @@ class TestTransactionSerialization(unittest.TestCase):
self.assertEqual(len(tx.outputs), 2)
coinbase = tx.inputs[0]
self.assertEqual(coinbase.output_txhash, NULL_HASH)
self.assertEqual(coinbase.output_index, 0xFFFFFFFF)
self.assertTrue(coinbase.txo_ref.is_null)
self.assertEqual(coinbase.txo_ref.position, 0xFFFFFFFF)
self.assertEqual(coinbase.sequence, 4294967295)
self.assertTrue(coinbase.is_coinbase)
self.assertEqual(coinbase.script, None)
self.assertEqual(
coinbase.coinbase[9:22],
b'/BTC.COM/NYA/'
)
self.assertIsNotNone(coinbase.coinbase)
self.assertIsNone(coinbase.script)
self.assertEqual(coinbase.coinbase[9:22], b'/BTC.COM/NYA/')
out = tx.outputs[0]
self.assertEqual(out.amount, 1561039505)
self.assertEqual(out.index, 0)
self.assertEqual(out.position, 0)
self.assertFalse(out.script.is_pay_pubkey)
self.assertFalse(out.script.is_pay_pubkey_hash)
self.assertTrue(out.script.is_pay_script_hash)
@ -126,7 +123,7 @@ class TestTransactionSerialization(unittest.TestCase):
out1 = tx.outputs[1]
self.assertEqual(out1.amount, 0)
self.assertEqual(out1.index, 1)
self.assertEqual(out1.position, 1)
self.assertEqual(
hexlify(out1.script.values['data']),
b'aa21a9ede6c99265a6b9e1d36c962fda0516b35709c49dc3b8176fa7e5d5f1f6197884b4'
@ -155,19 +152,20 @@ class TestTransactionSigning(unittest.TestCase):
)
yield account.ensure_address_gap()
address1 = (yield account.receiving.get_addresses())[0]
address2 = (yield account.receiving.get_addresses())[0]
address1, address2 = yield account.receiving.get_addresses(2)
pubkey_hash1 = self.ledger.address_to_hash160(address1)
pubkey_hash2 = self.ledger.address_to_hash160(address2)
tx = ledger_class.transaction_class() \
.add_inputs([ledger_class.transaction_class.input_class.spend(get_output(2*COIN, pubkey_hash1))]) \
.add_outputs([ledger_class.transaction_class.output_class.pay_pubkey_hash(int(1.9*COIN), pubkey_hash2)]) \
tx_class = ledger_class.transaction_class
tx = tx_class() \
.add_inputs([tx_class.input_class.spend(get_output(2*COIN, pubkey_hash1))]) \
.add_outputs([tx_class.output_class.pay_pubkey_hash(int(1.9*COIN), pubkey_hash2)]) \
yield tx.sign([account])
self.assertEqual(
hexlify(tx.inputs[0].script.values['signature']),
b'3044022064cd6b95c9e0084253c10dd56bcec2bfd816c29aad05fbea490511d79540462b02201aa9d6f73'
b'48bb0c76b28d1ad87cf4ffd51cf4de0b299af8bf0ecad70e3369ef201'
b'304402203d463519290d06891e461ea5256c56097ccdad53379b1bb4e51ec5abc6e9fd02022034ed15b9'
b'd7c678716c4aa7c0fd26c688e8f9db8075838f2839ab55d551b62c0a01'
)

6
tests/unit/test_wallet.py
View file

@ -32,6 +32,7 @@ class TestWalletCreation(unittest.TestCase):
'name': 'Main Wallet',
'accounts': [
{
'name': 'An Account',
'ledger': 'btc_mainnet',
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
@ -43,10 +44,11 @@ class TestWalletCreation(unittest.TestCase):
'public_key':
'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f'
'iW44g14WF52fYC5J483wqQ5ZP',
'is_hd': True,
'receiving_gap': 10,
'receiving_maximum_use_per_address': 2,
'receiving_maximum_uses_per_address': 2,
'change_gap': 10,
'change_maximum_use_per_address': 2,
'change_maximum_uses_per_address': 2,
}
]
}

6
torba/baseaccount.py
View file

@ -200,7 +200,7 @@ class BaseAccount(object):
def to_dict(self):
private_key = self.private_key
if not self.encrypted and self.private_key:
private_key = self.private_key.extended_key_string().decode()
private_key = self.private_key.extended_key_string()
d = {
'ledger': self.ledger.get_id(),
@ -208,7 +208,7 @@ class BaseAccount(object):
'seed': self.seed,
'encrypted': self.encrypted,
'private_key': private_key,
'public_key': self.public_key.extended_key_string().decode(),
'public_key': self.public_key.extended_key_string(),
'is_hd': False
}
@ -260,7 +260,7 @@ class BaseAccount(object):
else:
return self.private_key.child(chain).child(index)
def get_balance(self, confirmations, **constraints):
def get_balance(self, confirmations=6, **constraints):
if confirmations == 0:
return self.ledger.db.get_balance_for_account(self, **constraints)
else:

68
torba/basedatabase.py
View file

@ -6,7 +6,7 @@ import sqlite3
from twisted.internet import defer
from twisted.enterprise import adbapi
import torba.baseaccount
from torba.hash import TXRefImmutable
log = logging.getLogger(__name__)
@ -131,8 +131,8 @@ class BaseDatabase(SQLiteMixin):
CREATE_TXO_TABLE = """
create table if not exists txo (
txoid text primary key,
txid text references tx,
txoid text primary key,
address text references pubkey_address,
position integer not null,
amount integer not null,
@ -158,8 +158,9 @@ class BaseDatabase(SQLiteMixin):
def txo_to_row(self, tx, address, txo):
return {
'txid': sqlite3.Binary(tx.hash),
'address': sqlite3.Binary(address),
'txid': tx.id,
'txoid': txo.id,
'address': address,
'position': txo.position,
'amount': txo.amount,
'script': sqlite3.Binary(txo.script.source)
@ -170,7 +171,7 @@ class BaseDatabase(SQLiteMixin):
def _steps(t):
if save_tx == 'insert':
t.execute(*self._insert_sql('tx', {
'txhash': sqlite3.Binary(tx.hash),
'txid': tx.id,
'raw': sqlite3.Binary(tx.raw),
'height': height,
'is_verified': is_verified
@ -178,16 +179,15 @@ class BaseDatabase(SQLiteMixin):
elif save_tx == 'update':
t.execute(*self._update_sql("tx", {
'height': height, 'is_verified': is_verified
}, 'txhash = ?', (sqlite3.Binary(tx.hash),)
}, 'txid = ?', (tx.id,)
))
existing_txos = list(map(itemgetter(0), t.execute(
"SELECT position FROM txo WHERE txhash = ?",
(sqlite3.Binary(tx.hash),)
"SELECT position FROM txo WHERE txid = ?", (tx.id,)
).fetchall()))
for txo in tx.outputs:
if txo.index in existing_txos:
if txo.position in existing_txos:
continue
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == hash:
t.execute(*self._insert_sql("txo", self.txo_to_row(tx, address, txo)))
@ -195,20 +195,17 @@ class BaseDatabase(SQLiteMixin):
# TODO: implement script hash payments
print('Database.save_transaction_io: pay script hash is not implemented!')
existing_txis = [txi[0] for txi in t.execute(
"SELECT txoid FROM txi WHERE txhash = ? AND address = ?",
(sqlite3.Binary(tx.hash), sqlite3.Binary(address))).fetchall()]
spent_txoids = [txi[0] for txi in t.execute(
"SELECT txoid FROM txi WHERE txid = ? AND address = ?", (tx.id, address)
).fetchall()]
for txi in tx.inputs:
txoid = t.execute(
"SELECT txoid FROM txo WHERE txhash = ? AND position = ?",
(sqlite3.Binary(txi.output_txhash), txi.output_index)
).fetchone()
if txoid is not None and txoid[0] not in existing_txis:
txoid = txi.txo_ref.id
if txoid not in spent_txoids:
t.execute(*self._insert_sql("txi", {
'txhash': sqlite3.Binary(tx.hash),
'address': sqlite3.Binary(address),
'txoid': txoid[0],
'txid': tx.id,
'txoid': txoid,
'address': address,
}))
self._set_address_history(t, address, history)
@ -225,16 +222,10 @@ class BaseDatabase(SQLiteMixin):
def release_reserved_outputs(self, txoids):
return self.reserve_spent_outputs(txoids, is_reserved=False)
def get_txoid_for_txo(self, txo):
return self.query_one_value(
"SELECT txoid FROM txo WHERE txhash = ? AND position = ?",
(sqlite3.Binary(txo.transaction.hash), txo.index)
)
@defer.inlineCallbacks
def get_transaction(self, txhash):
def get_transaction(self, txid):
result = yield self.db.runQuery(
"SELECT raw, height, is_verified FROM tx WHERE txhash = ?", (sqlite3.Binary(txhash),)
"SELECT raw, height, is_verified FROM tx WHERE txid = ?", (txid,)
)
if result:
defer.returnValue(result[0])
@ -254,13 +245,13 @@ class BaseDatabase(SQLiteMixin):
col, op = key[:-len('__lte')], '<='
extras.append('{} {} :{}'.format(col, op, key))
extra_sql = ' AND ' + ' AND '.join(extras)
values = {'account': sqlite3.Binary(account.public_key.address)}
values = {'account': account.public_key.address}
values.update(constraints)
result = yield self.db.runQuery(
"""
SELECT SUM(amount)
FROM txo
JOIN tx ON tx.txhash=txo.txhash
JOIN tx ON tx.txid=txo.txid
JOIN pubkey_address ON pubkey_address.address=txo.address
WHERE
pubkey_address.account=:account AND
@ -279,11 +270,11 @@ class BaseDatabase(SQLiteMixin):
extra_sql = ' AND ' + ' AND '.join(
'{} = :{}'.format(c, c) for c in constraints.keys()
)
values = {'account': sqlite3.Binary(account.public_key.address)}
values = {'account': account.public_key.address}
values.update(constraints)
utxos = yield self.db.runQuery(
"""
SELECT amount, script, txhash, txo.position, txoid
SELECT amount, script, txid, txo.position
FROM txo JOIN pubkey_address ON pubkey_address.address=txo.address
WHERE account=:account AND txo.is_reserved=0 AND txoid NOT IN (SELECT txoid FROM txi)
"""+extra_sql, values
@ -293,9 +284,8 @@ class BaseDatabase(SQLiteMixin):
output_class(
values[0],
output_class.script_class(values[1]),
values[2],
index=values[3],
txoid=values[4]
TXRefImmutable.from_id(values[2]),
position=values[3]
) for values in utxos
])
@ -307,18 +297,18 @@ class BaseDatabase(SQLiteMixin):
) + ', '.join(['(?, ?, ?, ?, ?)'] * len(keys))
values = []
for position, pubkey in keys:
values.append(sqlite3.Binary(pubkey.address))
values.append(sqlite3.Binary(account.public_key.address))
values.append(pubkey.address)
values.append(account.public_key.address)
values.append(chain)
values.append(position)
values.append(sqlite3.Binary(pubkey.pubkey_bytes))
values.append(pubkey.pubkey_bytes)
return self.db.runOperation(sql, values)
@staticmethod
def _set_address_history(t, address, history):
t.execute(
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
(history, history.count(':')//2, sqlite3.Binary(address))
(history, history.count(':')//2, address)
)
def set_address_history(self, address, history):

14
torba/baseledger.py
View file

@ -77,7 +77,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)):
self.on_transaction = self._on_transaction_controller.stream
self.on_transaction.listen(
lambda e: log.info('({}) on_transaction: address={}, height={}, is_verified={}, tx.id={}'.format(
self.get_id(), e.address, e.height, e.is_verified, e.tx.hex_id)
self.get_id(), e.address, e.height, e.is_verified, e.tx.id)
)
)
@ -136,7 +136,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)):
match = yield self.db.get_address(address)
if match:
for account in self.accounts:
if bytes(match['account']) == account.public_key.address:
if match['account'] == account.public_key.address:
defer.returnValue(account.get_private_key(match['chain'], match['position']))
@defer.inlineCallbacks
@ -178,7 +178,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)):
@defer.inlineCallbacks
def is_valid_transaction(self, tx, height):
height <= len(self.headers) or defer.returnValue(False)
merkle = yield self.network.get_merkle(tx.hex_id.decode(), height)
merkle = yield self.network.get_merkle(tx.id, height)
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = self.headers[height]
defer.returnValue(merkle_root == header['merkle_root'])
@ -262,7 +262,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)):
synced_history.append((hex_id, remote_height))
if i < len(local_history) and local_history[i] == (hex_id.decode(), remote_height):
if i < len(local_history) and local_history[i] == (hex_id, remote_height):
continue
lock = self._transaction_processing_locks.setdefault(hex_id, defer.DeferredLock())
@ -271,7 +271,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)):
#try:
# see if we have a local copy of transaction, otherwise fetch it from server
raw, local_height, is_verified = yield self.db.get_transaction(unhexlify(hex_id)[::-1])
raw, local_height, is_verified = yield self.db.get_transaction(hex_id)
save_tx = None
if raw is None:
_raw = yield self.network.get_transaction(hex_id)
@ -288,7 +288,7 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)):
yield self.db.save_transaction_io(
save_tx, tx, remote_height, is_verified, address, self.address_to_hash160(address),
''.join('{}:{}:'.format(tx_id.decode(), tx_height) for tx_id, tx_height in synced_history)
''.join('{}:{}:'.format(tx_id, tx_height) for tx_id, tx_height in synced_history)
)
log.debug("{}: sync'ed tx {} for address: {}, height: {}, verified: {}".format(
@ -320,4 +320,4 @@ class BaseLedger(six.with_metaclass(LedgerRegistry)):
yield self.update_history(address)
def broadcast(self, tx):
return self.network.broadcast(hexlify(tx.raw))
return self.network.broadcast(hexlify(tx.raw).decode())

37
torba/basenetwork.py
View file

@ -1,4 +1,3 @@
import six
import json
import socket
import logging
@ -13,27 +12,6 @@ from torba.stream import StreamController
log = logging.getLogger(__name__)
if six.PY3:
buffer = memoryview
def unicode2bytes(string):
if isinstance(string, six.text_type):
return string.encode('iso-8859-1')
elif isinstance(string, list):
return [unicode2bytes(s) for s in string]
return string
def bytes2unicode(maybe_bytes):
if isinstance(maybe_bytes, buffer):
maybe_bytes = str(maybe_bytes)
if isinstance(maybe_bytes, bytes):
return maybe_bytes.decode()
elif isinstance(maybe_bytes, (list, tuple)):
return [bytes2unicode(b) for b in maybe_bytes]
return maybe_bytes
class StratumClientProtocol(LineOnlyReceiver):
delimiter = b'\n'
@ -86,14 +64,7 @@ class StratumClientProtocol(LineOnlyReceiver):
log.debug('received: {}'.format(line))
try:
# `line` comes in as a byte string but `json.loads` automatically converts everything to
# unicode. For keys it's not a big deal but for values there is an expectation
# everywhere else in wallet code that most values are byte strings.
message = json.loads(
line, object_hook=lambda obj: {
k: unicode2bytes(v) for k, v in obj.items()
}
)
message = json.loads(line)
except (ValueError, TypeError):
raise ValueError("Cannot decode message '{}'".format(line.strip()))
@ -118,7 +89,7 @@ class StratumClientProtocol(LineOnlyReceiver):
message = json.dumps({
'id': message_id,
'method': method,
'params': [bytes2unicode(arg) for arg in args]
'params': args
})
log.debug('sent: {}'.format(message))
self.sendLine(message.encode('latin-1'))
@ -160,8 +131,8 @@ class BaseNetwork:
self.on_status = self._on_status_controller.stream
self.subscription_controllers = {
b'blockchain.headers.subscribe': self._on_header_controller,
b'blockchain.address.subscribe': self._on_status_controller,
'blockchain.headers.subscribe': self._on_header_controller,
'blockchain.address.subscribe': self._on_status_controller,
}
@defer.inlineCallbacks

173
torba/basetransaction.py
View file

@ -9,30 +9,60 @@ import torba.baseaccount
import torba.baseledger
from torba.basescript import BaseInputScript, BaseOutputScript
from torba.coinselection import CoinSelector
from torba.constants import COIN
from torba.constants import COIN, NULL_HASH32
from torba.bcd_data_stream import BCDataStream
from torba.hash import sha256
from torba.hash import sha256, TXRef, TXRefImmutable, TXORef
from torba.util import ReadOnlyList
log = logging.getLogger()
NULL_HASH = b'\x00'*32
class TXRefMutable(TXRef):
__slots__ = 'tx',
def __init__(self, tx):
super(TXRefMutable, self).__init__()
self.tx = tx
@property
def id(self):
if self._id is None:
self._id = hexlify(self.hash[::-1]).decode()
return self._id
@property
def hash(self):
if self._hash is None:
self._hash = sha256(sha256(self.tx.raw))
return self._hash
def reset(self):
self._id = None
self._hash = None
class TXORefResolvable(TXORef):
__slots__ = '_txo',
def __init__(self, txo):
super(TXORefResolvable, self).__init__(txo.tx_ref, txo.position)
self._txo = txo
@property
def txo(self):
return self._txo
class InputOutput(object):
def __init__(self, txhash, index=None):
self._txhash = txhash # type: bytes
self.transaction = None # type: BaseTransaction
self.index = index # type: int
__slots__ = 'tx_ref', 'position'
@property
def txhash(self):
if self._txhash is None:
self._txhash = self.transaction.hash
return self._txhash
def __init__(self, tx_ref=None, position=None):
self.tx_ref = tx_ref # type: TXRef
self.position = position # type: int
@property
def size(self):
@ -52,49 +82,49 @@ class BaseInput(InputOutput):
NULL_SIGNATURE = b'\x00'*72
NULL_PUBLIC_KEY = b'\x00'*33
def __init__(self, output_or_txhash_index, script, sequence=0xFFFFFFFF, txhash=None):
super(BaseInput, self).__init__(txhash)
if isinstance(output_or_txhash_index, BaseOutput):
self.output = output_or_txhash_index # type: BaseOutput
self.output_txhash = self.output.txhash
self.output_index = self.output.index
else:
self.output = None # type: BaseOutput
self.output_txhash, self.output_index = output_or_txhash_index
__slots__ = 'txo_ref', 'sequence', 'coinbase', 'script'
def __init__(self, txo_ref, script, sequence=0xFFFFFFFF, tx_ref=None, position=None):
# type: (TXORef, BaseInputScript, int, TXRef, int) -> None
super(BaseInput, self).__init__(tx_ref, position)
self.txo_ref = txo_ref
self.sequence = sequence
self.is_coinbase = self.output_txhash == NULL_HASH
self.coinbase = script if self.is_coinbase else None
self.script = script if not self.is_coinbase else None # type: BaseInputScript
self.coinbase = script if txo_ref.is_null else None
self.script = script if not txo_ref.is_null else None # type: BaseInputScript
@property
def is_coinbase(self):
return self.coinbase is not None
@classmethod
def spend(cls, output):
def spend(cls, txo): # type: (BaseOutput) -> BaseInput
""" Create an input to spend the output."""
assert output.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.'
assert txo.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.'
script = cls.script_class.redeem_pubkey_hash(cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY)
return cls(output, script)
return cls(txo.ref, script)
@property
def amount(self):
""" Amount this input adds to the transaction. """
if self.output is None:
raise ValueError('Cannot get input value without referenced output.')
return self.output.amount
if self.txo_ref.txo is None:
raise ValueError('Cannot resolve output to get amount.')
return self.txo_ref.txo.amount
@classmethod
def deserialize_from(cls, stream):
txhash = stream.read(32)
index = stream.read_uint32()
tx_ref = TXRefImmutable.from_hash(stream.read(32))
position = stream.read_uint32()
script = stream.read_string()
sequence = stream.read_uint32()
return cls(
(txhash, index),
cls.script_class(script) if not txhash == NULL_HASH else script,
TXORef(tx_ref, position),
cls.script_class(script) if not tx_ref.is_null else script,
sequence
)
def serialize_to(self, stream, alternate_script=None):
stream.write(self.output_txhash)
stream.write_uint32(self.output_index)
stream.write(self.txo_ref.tx_ref.hash)
stream.write_uint32(self.txo_ref.position)
if alternate_script is not None:
stream.write_string(alternate_script)
else:
@ -107,7 +137,7 @@ class BaseInput(InputOutput):
class BaseOutputEffectiveAmountEstimator(object):
__slots__ = 'coin', 'txi', 'txo', 'fee', 'effective_amount'
__slots__ = 'txo', 'txi', 'fee', 'effective_amount'
def __init__(self, ledger, txo): # type: (torba.baseledger.BaseLedger, BaseOutput) -> None
self.txo = txo
@ -124,11 +154,21 @@ class BaseOutput(InputOutput):
script_class = BaseOutputScript
estimator_class = BaseOutputEffectiveAmountEstimator
def __init__(self, amount, script, txhash=None, index=None, txoid=None):
super(BaseOutput, self).__init__(txhash, index)
self.amount = amount # type: int
self.script = script # type: BaseOutputScript
self.txoid = txoid
__slots__ = 'amount', 'script'
def __init__(self, amount, script, tx_ref=None, position=None):
# type: (int, BaseOutputScript, TXRef, int) -> None
super(BaseOutput, self).__init__(tx_ref, position)
self.amount = amount
self.script = script
@property
def ref(self):
return TXORefResolvable(self)
@property
def id(self):
return self.ref.id
def get_estimator(self, ledger):
return self.estimator_class(ledger, self)
@ -156,8 +196,7 @@ class BaseTransaction:
def __init__(self, raw=None, version=1, locktime=0):
self._raw = raw
self._hash = None
self._id = None
self.ref = TXRefMutable(self)
self.version = version # type: int
self.locktime = locktime # type: int
self._inputs = [] # type: List[BaseInput]
@ -165,21 +204,13 @@ class BaseTransaction:
if raw is not None:
self._deserialize()
@property
def hex_id(self):
return hexlify(self.id)
@property
def id(self):
if self._id is None:
self._id = self.hash[::-1]
return self._id
return self.ref.id
@property
def hash(self):
if self._hash is None:
self._hash = sha256(sha256(self.raw))
return self._hash
return self.ref.hash
@property
def raw(self):
@ -188,9 +219,8 @@ class BaseTransaction:
return self._raw
def _reset(self):
self._id = None
self._hash = None
self._raw = None
self.ref.reset()
@property
def inputs(self): # type: () -> ReadOnlyList[BaseInput]
@ -201,35 +231,36 @@ class BaseTransaction:
return ReadOnlyList(self._outputs)
def _add(self, new_ios, existing_ios):
# type: (List[InputOutput], List[InputOutput]) -> BaseTransaction
for txio in new_ios:
txio.transaction = self
txio.index = len(existing_ios)
txio.tx_ref = self.ref
txio.position = len(existing_ios)
existing_ios.append(txio)
self._reset()
return self
def add_inputs(self, inputs):
def add_inputs(self, inputs): # type: (List[BaseInput]) -> BaseTransaction
return self._add(inputs, self._inputs)
def add_outputs(self, outputs):
def add_outputs(self, outputs): # type: (List[BaseOutput]) -> BaseTransaction
return self._add(outputs, self._outputs)
@property
def fee(self):
def fee(self): # type: () -> int
""" Fee that will actually be paid."""
return self.input_sum - self.output_sum
@property
def size(self):
def size(self): # type: () -> int
""" Size in bytes of the entire transaction. """
return len(self.raw)
@property
def base_size(self):
def base_size(self): # type: () -> int
""" Size in bytes of transaction meta data and all outputs; without inputs. """
return len(self._serialize(with_inputs=False))
def _serialize(self, with_inputs=True):
def _serialize(self, with_inputs=True): # type: (bool) -> bytes
stream = BCDataStream()
stream.write_uint32(self.version)
if with_inputs:
@ -242,13 +273,13 @@ class BaseTransaction:
stream.write_uint32(self.locktime)
return stream.get_bytes()
def _serialize_for_signature(self, signing_input):
def _serialize_for_signature(self, signing_input): # type: (int) -> bytes
stream = BCDataStream()
stream.write_uint32(self.version)
stream.write_compact_size(len(self._inputs))
for i, txin in enumerate(self._inputs):
if signing_input == i:
txin.serialize_to(stream, txin.output.script.source)
txin.serialize_to(stream, txin.txo_ref.txo.script.source)
else:
txin.serialize_to(stream, b'')
stream.write_compact_size(len(self._outputs))
@ -300,7 +331,7 @@ class BaseTransaction:
selector = CoinSelector(
txos, amount,
ledger.get_input_output_fee(
cls.output_class.pay_pubkey_hash(COIN, NULL_HASH)
cls.output_class.pay_pubkey_hash(COIN, NULL_HASH32)
)
)
@ -308,7 +339,7 @@ class BaseTransaction:
if not spendables:
raise ValueError('Not enough funds to cover this transaction.')
reserved_outputs = [s.txo.txoid for s in spendables]
reserved_outputs = [s.txo.id for s in spendables]
if reserve_outputs:
yield ledger.db.reserve_spent_outputs(reserved_outputs)
@ -340,14 +371,14 @@ class BaseTransaction:
])
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
reserved_outputs = [utxo.txoid for utxo in assets]
reserved_outputs = [utxo.id for utxo in assets]
if reserve_outputs:
yield ledger.db.reserve_spent_outputs(reserved_outputs)
try:
cost_of_change = (
ledger.get_transaction_base_fee(tx) +
ledger.get_input_output_fee(cls.output_class.pay_pubkey_hash(COIN, NULL_HASH))
ledger.get_input_output_fee(cls.output_class.pay_pubkey_hash(COIN, NULL_HASH32))
)
liquidated_total = sum(utxo.amount for utxo in assets)
if liquidated_total > cost_of_change:
@ -372,7 +403,7 @@ class BaseTransaction:
def sign(self, funding_accounts): # type: (Iterable[torba.baseaccount.BaseAccount]) -> BaseTransaction
ledger = self.ensure_all_have_same_ledger(funding_accounts)
for i, txi in enumerate(self._inputs):
txo_script = txi.output.script
txo_script = txi.txo_ref.txo.script
if txo_script.is_pay_pubkey_hash:
address = ledger.hash160_to_address(txo_script.values['pubkey_hash'])
private_key = yield ledger.get_private_key_for_address(address)

6
torba/coinselection.py
View file

@ -21,14 +21,14 @@ class CoinSelector:
if six.PY3 and seed is not None:
self.random.seed(seed, version=1)
def select(self):
def select(self): # type: () -> List[torba.basetransaction.BaseOutputAmountEstimator]
if not self.txos:
return
if self.target > self.available:
return
return self.branch_and_bound() or self.single_random_draw()
def branch_and_bound(self):
def branch_and_bound(self): # type: () -> List[torba.basetransaction.BaseOutputAmountEstimator]
# see bitcoin implementation for more info:
# https://github.com/bitcoin/bitcoin/blob/master/src/wallet/coinselection.cpp
@ -84,7 +84,7 @@ class CoinSelector:
self.txos[i] for i, include in enumerate(best_selection) if include
]
def single_random_draw(self):
def single_random_draw(self): # type: () -> List[torba.basetransaction.BaseOutputAmountEstimator]
self.random.shuffle(self.txos, self.random.random)
selection = []
amount = 0

1
torba/constants.py
View file

@ -1,3 +1,4 @@
NULL_HASH32 = b'\x00'*32
CENT = 1000000
COIN = 100*CENT

64
torba/hash.py
View file

@ -20,6 +20,7 @@ from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.backends import default_backend
from torba.util import bytes_to_int, int_to_bytes
from torba.constants import NULL_HASH32
_sha256 = hashlib.sha256
_sha512 = hashlib.sha512
@ -27,6 +28,67 @@ _new_hash = hashlib.new
_new_hmac = hmac.new
class TXRef(object):
__slots__ = '_id', '_hash'
def __init__(self):
self._id = None
self._hash = None
@property
def id(self):
return self._id
@property
def hash(self):
return self._hash
@property
def is_null(self):
return self.hash == NULL_HASH32
class TXRefImmutable(TXRef):
__slots__ = ()
@classmethod
def from_hash(cls, tx_hash): # type: (bytes) -> TXRefImmutable
ref = cls()
ref._hash = tx_hash
ref._id = hexlify(tx_hash[::-1]).decode()
return ref
@classmethod
def from_id(cls, tx_id): # type: (str) -> TXRefImmutable
ref = cls()
ref._id = tx_id
ref._hash = unhexlify(tx_id)[::-1]
return ref
class TXORef(object):
__slots__ = 'tx_ref', 'position'
def __init__(self, tx_ref, position): # type: (TXRef, int) -> None
self.tx_ref = tx_ref
self.position = position
@property
def id(self):
return '{}:{}'.format(self.tx_ref.id, self.position)
@property
def is_null(self):
return self.tx_ref.is_null
@property
def txo(self):
return None
def sha256(x):
""" Simple wrapper of hashlib sha256. """
return _sha256(x).digest()
@ -164,7 +226,7 @@ class Base58(object):
break
txt += u'1'
return txt[::-1].encode()
return txt[::-1]
@classmethod
def decode_check(cls, txt, hash_fn=double_sha256):

18
torba/manager.py
View file

@ -1,8 +1,10 @@
from decimal import Decimal
from typing import List, Dict, Type
from twisted.internet import defer
from torba.baseledger import BaseLedger, LedgerRegistry
from torba.wallet import Wallet, WalletStorage
from torba.constants import COIN
class WalletManager(object):
@ -39,12 +41,18 @@ class WalletManager(object):
return wallet
@defer.inlineCallbacks
def get_balance(self):
def get_balances(self, confirmations=6):
balances = {}
for ledger in self.ledgers.values():
for account in ledger.accounts:
balances.setdefault(ledger.get_id(), 0)
balances[ledger.get_id()] += yield account.get_balance()
for i, ledger in enumerate(self.ledgers.values()):
ledger_balances = balances[ledger.get_id()] = []
for j, account in enumerate(ledger.accounts):
satoshis = yield account.get_balance(confirmations)
ledger_balances.append({
'account': account.name,
'coins': round(Decimal(satoshis) / COIN, 2),
'satoshis': satoshis,
'is_default_account': i == 0 and j == 0
})
defer.returnValue(balances)
@property