diff --git a/lbrynet/tests/integration/test_wallet.py b/lbrynet/tests/integration/test_wallet.py new file mode 100644 index 000000000..dc7e15b56 --- /dev/null +++ b/lbrynet/tests/integration/test_wallet.py @@ -0,0 +1,88 @@ +import time +import shutil +import logging +import tempfile +from binascii import hexlify + +from twisted.internet import defer, reactor, threads +from twisted.trial import unittest +from orchstr8.wrapper import BaseLbryServiceStack + +from lbrynet.core.call_later_manager import CallLaterManager +from lbrynet.database.storage import SQLiteStorage + +from lbrynet.wallet import set_wallet_manager +from lbrynet.wallet.wallet import Wallet +from lbrynet.wallet.manager import WalletManager +from lbrynet.wallet.transaction import Transaction, Output +from lbrynet.wallet.constants import COIN, REGTEST_CHAIN +from lbrynet.wallet.hash import hash160_to_address, address_to_hash_160 + + +class WalletTestCase(unittest.TestCase): + + VERBOSE = False + + def setUp(self): + logging.getLogger('lbrynet').setLevel(logging.INFO) + self.data_path = tempfile.mkdtemp() + self.db = SQLiteStorage(self.data_path) + self.config = { + 'chain': REGTEST_CHAIN, + 'wallet_path': self.data_path, + 'default_servers': [('localhost', 50001)] + } + CallLaterManager.setup(reactor.callLater) + self.service = BaseLbryServiceStack(self.VERBOSE) + return self.service.startup() + + def tearDown(self): + CallLaterManager.stop() + shutil.rmtree(self.data_path, ignore_errors=True) + return self.service.shutdown() + + @property + def lbrycrd(self): + return self.service.lbrycrd + + +class StartupTests(WalletTestCase): + + VERBOSE = True + + @defer.inlineCallbacks + def test_balance(self): + wallet = Wallet(chain=REGTEST_CHAIN) + manager = WalletManager(self.config, wallet) + set_wallet_manager(manager) + yield manager.start() + yield self.lbrycrd.generate(1) + yield threads.deferToThread(time.sleep, 1) + #yield wallet.network.on_header.first + address = manager.get_least_used_receiving_address() + sendtxid = yield self.lbrycrd.sendtoaddress(address, 2.5) + yield self.lbrycrd.generate(1) + #yield manager.wallet.history.on_transaction. + yield threads.deferToThread(time.sleep, 10) + tx = manager.ledger.transactions.values()[0] + print(tx.to_python_source()) + print(address) + output = None + for txo in tx.outputs: + other = hash160_to_address(txo.script.values['pubkey_hash'], 'regtest') + if other == address: + output = txo + break + + address2 = manager.get_least_used_receiving_address() + tx = Transaction() + tx.add_inputs([output.spend()]) + Output.pay_pubkey_hash(tx, 0, 2.49*COIN, address_to_hash_160(address2)) + print(tx.to_python_source()) + tx.sign(wallet) + print(tx.to_python_source()) + + yield self.lbrycrd.decoderawtransaction(hexlify(tx.raw)) + yield self.lbrycrd.sendrawtransaction(hexlify(tx.raw)) + + yield manager.stop() diff --git a/lbrynet/tests/unit/txlbryum/__init__.py b/lbrynet/tests/unit/txlbryum/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbrynet/tests/unit/wallet/test_script.py b/lbrynet/tests/unit/wallet/test_script.py index 1a1b1ccf6..c4ef366fa 100644 --- a/lbrynet/tests/unit/wallet/test_script.py +++ b/lbrynet/tests/unit/wallet/test_script.py @@ -71,7 +71,7 @@ class TestScriptTemplates(unittest.TestCase): def test_push_data_mixed(self): self.assertEqual(parse( - (PUSH_SINGLE('CEO'), PUSH_MANY('Devs'), PUSH_SINGLE(b'CTO'), PUSH_SINGLE(b'State')), + (PUSH_SINGLE('CEO'), PUSH_MANY('Devs'), PUSH_SINGLE('CTO'), PUSH_SINGLE('State')), (b'jeremy', b'lex', b'amit', b'victor', b'jack', b'grin', b'NH') ), { 'CEO': b'jeremy', @@ -114,7 +114,8 @@ class TestRedeemPubKeyHash(unittest.TestCase): def test_redeem_pubkey_hash_1(self): self.assertEqual( self.redeem_pubkey_hash( - b'30450221009dc93f25184a8d483745cd3eceff49727a317c9bfd8be8d3d04517e9cdaf8dd502200e02dc5939cad9562d2b1f303f185957581c4851c98d497af281118825e18a8301', + b'30450221009dc93f25184a8d483745cd3eceff49727a317c9bfd8be8d3d04517e9cdaf8dd502200e' + b'02dc5939cad9562d2b1f303f185957581c4851c98d497af281118825e18a8301', b'025415a06514230521bff3aaface31f6db9d9bbc39bf1ca60a189e78731cfd4e1b' ), '4830450221009dc93f25184a8d483745cd3eceff49727a317c9bfd8be8d3d04517e9cdaf8dd502200e02d' diff --git a/lbrynet/tests/unit/wallet/test_transaction.py b/lbrynet/tests/unit/wallet/test_transaction.py new file mode 100644 index 000000000..22268e4db --- /dev/null +++ b/lbrynet/tests/unit/wallet/test_transaction.py @@ -0,0 +1,229 @@ +from binascii import hexlify, unhexlify +from twisted.trial import unittest +from lbrynet.wallet.constants import CENT +from lbrynet.wallet.transaction import Transaction, Input, Output +from lbrynet.wallet.manager import WalletManager +from lbrynet.wallet import set_wallet_manager +from lbrynet.wallet.bip32 import PrivateKey +from lbrynet.wallet.mnemonic import Mnemonic + + +NULL_HASH = '\x00'*32 +FEE_PER_BYTE = 50 +FEE_PER_CHAR = 200000 + + +class TestSizeAndFeeEstimation(unittest.TestCase): + + def setUp(self): + set_wallet_manager(WalletManager({ + 'fee_per_byte': FEE_PER_BYTE, + 'fee_per_name_char': FEE_PER_CHAR + })) + + @staticmethod + def get_output(): + return Output.pay_pubkey_hash(Transaction(), 1, CENT, NULL_HASH) + + @classmethod + def get_input(cls): + return cls.get_output().spend(fake=True) + + @classmethod + def get_transaction(cls): + tx = Transaction() + Output.pay_pubkey_hash(tx, 1, CENT, NULL_HASH) + tx.add_inputs([cls.get_input()]) + return tx + + @classmethod + def get_claim_transaction(cls, claim_name, claim=''): + tx = Transaction() + Output.pay_claim_name_pubkey_hash(tx, 1, CENT, claim_name, claim, NULL_HASH) + tx.add_inputs([cls.get_input()]) + return tx + + def test_output_size_and_fee(self): + txo = self.get_output() + self.assertEqual(txo.size, 46) + self.assertEqual(txo.fee, 46 * FEE_PER_BYTE) + + def test_input_size_and_fee(self): + txi = self.get_input() + self.assertEqual(txi.size, 148) + self.assertEqual(txi.fee, 148 * FEE_PER_BYTE) + + def test_transaction_size_and_fee(self): + tx = self.get_transaction() + base_size = tx.size - 1 - tx.inputs[0].size + self.assertEqual(tx.size, 204) + self.assertEqual(tx.base_size, base_size) + self.assertEqual(tx.base_fee, FEE_PER_BYTE * base_size) + + def test_claim_name_transaction_size_and_fee(self): + # fee based on claim name is the larger fee + claim_name = 'verylongname' + tx = self.get_claim_transaction(claim_name, '0'*4000) + base_size = tx.size - 1 - tx.inputs[0].size + self.assertEqual(tx.size, 4225) + self.assertEqual(tx.base_size, base_size) + self.assertEqual(tx.base_fee, len(claim_name) * FEE_PER_CHAR) + # fee based on total bytes is the larger fee + claim_name = 'a' + tx = self.get_claim_transaction(claim_name, '0'*4000) + base_size = tx.size - 1 - tx.inputs[0].size + self.assertEqual(tx.size, 4214) + self.assertEqual(tx.base_size, base_size) + self.assertEqual(tx.base_fee, FEE_PER_BYTE * base_size) + + +class TestTransactionSerialization(unittest.TestCase): + + def test_genesis_transaction(self): + raw = unhexlify( + "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1f0" + "4ffff001d010417696e736572742074696d657374616d7020737472696e67ffffffff01000004bfc91b8e" + "001976a914345991dbf57bfb014b87006acdfafbfc5fe8292f88ac00000000" + ) + tx = Transaction(raw) + self.assertEqual(tx.version, 1) + self.assertEqual(tx.locktime, 0) + self.assertEqual(len(tx.inputs), 1) + self.assertEqual(len(tx.outputs), 1) + + coinbase = tx.inputs[0] + self.assertEqual(coinbase.output_tx_hash, NULL_HASH) + self.assertEqual(coinbase.output_index, 0xFFFFFFFF) + self.assertEqual(coinbase.sequence, 0xFFFFFFFF) + self.assertTrue(coinbase.is_coinbase) + self.assertEqual(coinbase.script, None) + self.assertEqual( + hexlify(coinbase.coinbase), + b'04ffff001d010417696e736572742074696d657374616d7020737472696e67' + ) + + out = tx.outputs[0] + self.assertEqual(out.amount, 40000000000000000) + self.assertEqual(out.index, 0) + self.assertTrue(out.script.is_pay_pubkey_hash) + self.assertFalse(out.script.is_pay_script_hash) + self.assertFalse(out.script.is_claim_involved) + + tx._reset() + self.assertEqual(tx.raw, raw) + + def test_coinbase_transaction(self): + raw = unhexlify( + "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff200" + "34d520504f89ac55a086032d217bf0700000d2f6e6f64655374726174756d2f0000000001a03489850800" + "00001976a914cfab870d6deea54ca94a41912a75484649e52f2088ac00000000" + ) + tx = Transaction(raw) + self.assertEqual(tx.version, 1) + self.assertEqual(tx.locktime, 0) + self.assertEqual(len(tx.inputs), 1) + self.assertEqual(len(tx.outputs), 1) + + coinbase = tx.inputs[0] + self.assertEqual(coinbase.output_tx_hash, NULL_HASH) + self.assertEqual(coinbase.output_index, 0xFFFFFFFF) + self.assertEqual(coinbase.sequence, 0) + self.assertTrue(coinbase.is_coinbase) + self.assertEqual(coinbase.script, None) + self.assertEqual( + hexlify(coinbase.coinbase), + b'034d520504f89ac55a086032d217bf0700000d2f6e6f64655374726174756d2f' + ) + + out = tx.outputs[0] + self.assertEqual(out.amount, 36600100000) + self.assertEqual(out.index, 0) + self.assertTrue(out.script.is_pay_pubkey_hash) + self.assertFalse(out.script.is_pay_script_hash) + self.assertFalse(out.script.is_claim_involved) + + tx._reset() + self.assertEqual(tx.raw, raw) + + def test_claim_transaction(self): + raw = unhexlify( + "01000000012433e1b327603843b083344dbae5306ff7927f87ebbc5ae9eb50856c5b53fd1d000000006a4" + "7304402201a91e1023d11c383a11e26bf8f9034087b15d8ada78fa565e0610455ffc8505e0220038a63a6" + "ecb399723d4f1f78a20ddec0a78bf8fb6c75e63e166ef780f3944fbf0121021810150a2e4b088ec51b20c" + "be1b335962b634545860733367824d5dc3eda767dffffffff028096980000000000fdff00b50463617473" + "4cdc080110011a7808011230080410011a084d616361726f6e6922002a003214416c6c207269676874732" + "072657365727665642e38004a0052005a001a42080110011a30add80aaf02559ba09853636a0658c42b72" + "7cb5bb4ba8acedb4b7fe656065a47a31878dbf9912135ddb9e13806cc1479d220a696d6167652f6a70656" + "72a5c080110031a404180cc0fa4d3839ee29cca866baed25fafb43fca1eb3b608ee889d351d3573d042c7" + "b83e2e643db0d8e062a04e6e9ae6b90540a2f95fe28638d0f18af4361a1c2214f73de93f4299fb32c32f9" + "49e02198a8e91101abd6d7576a914be16e4b0f9bd8f6d47d02b3a887049c36d3b84cb88ac0cd2520b0000" + "00001976a914f521178feb733a719964e1da4a9efb09dcc39cfa88ac00000000" + ) + tx = Transaction(raw) + self.assertEqual(hexlify(tx.id), b'666c3d15de1d6949a4fe717126c368e274b36957dce29fd401138c1e87e92a62') + self.assertEqual(tx.version, 1) + self.assertEqual(tx.locktime, 0) + self.assertEqual(len(tx.inputs), 1) + self.assertEqual(len(tx.outputs), 2) + + txin = tx.inputs[0] # type: Input + self.assertEqual( + hexlify(txin.output_tx_hash[::-1]), + b'1dfd535b6c8550ebe95abceb877f92f76f30e5ba4d3483b043386027b3e13324' + ) + self.assertEqual(txin.output_index, 0) + self.assertEqual(txin.sequence, 0xFFFFFFFF) + self.assertFalse(txin.is_coinbase) + self.assertEqual(txin.script.template.name, 'pubkey_hash') + self.assertEqual( + hexlify(txin.script.values['pubkey']), + b'021810150a2e4b088ec51b20cbe1b335962b634545860733367824d5dc3eda767d' + ) + self.assertEqual( + hexlify(txin.script.values['signature']), + b'304402201a91e1023d11c383a11e26bf8f9034087b15d8ada78fa565e0610455ffc8505e0220038a63a6' + b'ecb399723d4f1f78a20ddec0a78bf8fb6c75e63e166ef780f3944fbf01' + ) + + # Claim + out0 = tx.outputs[0] # type: Output + self.assertEqual(out0.amount, 10000000) + self.assertEqual(out0.index, 0) + self.assertTrue(out0.script.is_pay_pubkey_hash) + self.assertTrue(out0.script.is_claim_name) + self.assertTrue(out0.script.is_claim_involved) + self.assertEqual(out0.script.values['claim_name'], b'cats') + self.assertEqual( + hexlify(out0.script.values['pubkey_hash']), + b'be16e4b0f9bd8f6d47d02b3a887049c36d3b84cb' + ) + + # Change + out1 = tx.outputs[1] # type: Output + self.assertEqual(out1.amount, 189977100) + self.assertEqual(out1.index, 1) + self.assertTrue(out1.script.is_pay_pubkey_hash) + self.assertFalse(out1.script.is_claim_involved) + self.assertEqual( + hexlify(out1.script.values['pubkey_hash']), + b'f521178feb733a719964e1da4a9efb09dcc39cfa' + ) + + tx._reset() + self.assertEqual(tx.raw, raw) + + +class TestTransactionSigning(unittest.TestCase): + + def setUp(self): + self.private_key = PrivateKey.from_seed(Mnemonic.mnemonic_to_seed( + 'program leader library giant team normal suspect crater pair miracle sweet until absent' + )) + + def test_sign(self): + tx = Transaction() + Output.pay_pubkey_hash(Transaction(), 0, CENT, NULL_HASH).spend(fake=True) + tx.add_inputs([self.get_input()]) + Output.pay_pubkey_hash(tx, 0, CENT, NULL_HASH) + tx = self.get_tx() + diff --git a/lbrynet/tests/unit/wallet/test_wallet.py b/lbrynet/tests/unit/wallet/test_wallet.py new file mode 100644 index 000000000..e58586f0b --- /dev/null +++ b/lbrynet/tests/unit/wallet/test_wallet.py @@ -0,0 +1,98 @@ +from twisted.trial import unittest +from lbrynet.wallet.wallet import Account, Wallet +from lbrynet.wallet.manager import WalletManager +from lbrynet.wallet import set_wallet_manager + + +class TestWalletAccount(unittest.TestCase): + + def test_wallet_automatically_creates_default_account(self): + wallet = Wallet() + set_wallet_manager(WalletManager(wallet=wallet)) + account = wallet.default_account # type: Account + self.assertIsInstance(account, Account) + self.assertEqual(len(account.receiving_keys.child_keys), 0) + self.assertEqual(len(account.receiving_keys.addresses), 0) + self.assertEqual(len(account.change_keys.child_keys), 0) + self.assertEqual(len(account.change_keys.addresses), 0) + wallet.ensure_enough_addresses() + self.assertEqual(len(account.receiving_keys.child_keys), 20) + self.assertEqual(len(account.receiving_keys.addresses), 20) + self.assertEqual(len(account.change_keys.child_keys), 6) + self.assertEqual(len(account.change_keys.addresses), 6) + + def test_generate_account_from_seed(self): + account = Account.generate_from_seed( + "carbon smart garage balance margin twelve chest sword toast envelope bottom stomach ab" + "sent" + ) # type: Account + self.assertEqual( + account.private_key.extended_key_string(), + "xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7DRNLEoB8HoirMgH969NrgL8jNzLEeg" + "qFzPRWM37GXd4uE8uuRkx4LAe", + ) + self.assertEqual( + account.public_key.extended_key_string(), + "xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMmDgp66FxHuDtWdft3B5eL5xQtyzAtk" + "dmhhC95gjRjLzSTdkho95asu9", + ) + self.assertEqual( + account.receiving_keys.generate_next_address(), + 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx' + ) + private_key = account.get_private_key_for_address('bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx') + self.assertEqual( + private_key.extended_key_string(), + 'xprv9vwXVierUTT4hmoe3dtTeBfbNv1ph2mm8RWXARU6HsZjBaAoFaS2FRQu4fptRAyJWhJW42dmsEaC1nKnVK' + 'KTMhq3TVEHsNj1ca3ciZMKktT' + ) + self.assertIsNone(account.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')) + + def test_load_and_save_account(self): + wallet_data = { + 'name': 'Main Wallet', + 'accounts': { + 0: { + 'seed': + "carbon smart garage balance margin twelve chest sword toast envelope botto" + "m stomach absent", + 'encrypted': False, + 'private_key': + "xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7DRNLEoB8HoirMgH969" + "NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe", + 'public_key': + "xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMmDgp66FxHuDtWdft3B" + "5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9", + 'receiving_gap': 10, + 'receiving_keys': [ + '02c68e2d1cf85404c86244ffa279f4c5cd00331e996d30a86d6e46480e3a9220f4', + '03c5a997d0549875d23b8e4bbc7b4d316d962587483f3a2e62ddd90a21043c4941'], + 'change_gap': 10, + 'change_keys': [ + '021460e8d728eee325d0d43128572b2e2bacdc027e420451df100cf9f2154ea5ab'] + } + } + } + + wallet = Wallet.from_json(wallet_data) + set_wallet_manager(WalletManager(wallet=wallet)) + self.assertEqual(wallet.name, 'Main Wallet') + + account = wallet.default_account + self.assertIsInstance(account, Account) + + self.assertEqual(len(account.receiving_keys.addresses), 2) + self.assertEqual( + account.receiving_keys.addresses[0], + 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx' + ) + self.assertEqual(len(account.change_keys.addresses), 1) + self.assertEqual( + account.change_keys.addresses[0], + 'bFpHENtqugKKHDshKFq2Mnb59Y2bx4vKgL' + ) + + self.assertDictEqual( + wallet_data['accounts'][0], + account.to_json() + ) diff --git a/lbrynet/wallet/__init__.py b/lbrynet/wallet/__init__.py index e69de29bb..7b8ba2a7a 100644 --- a/lbrynet/wallet/__init__.py +++ b/lbrynet/wallet/__init__.py @@ -0,0 +1,10 @@ +_wallet_manager = None + + +def set_wallet_manager(wallet_manager): + global _wallet_manager + _wallet_manager = wallet_manager + + +def get_wallet_manager(): + return _wallet_manager diff --git a/lbrynet/wallet/account.py b/lbrynet/wallet/account.py index a2db39599..12457e7a9 100644 --- a/lbrynet/wallet/account.py +++ b/lbrynet/wallet/account.py @@ -1,75 +1,141 @@ -import logging +from binascii import hexlify, unhexlify +from itertools import chain +from lbrynet.wallet import get_wallet_manager +from lbrynet.wallet.mnemonic import Mnemonic +from lbrynet.wallet.bip32 import PrivateKey, PubKey, from_extended_key_string +from lbrynet.wallet.hash import double_sha256, aes_encrypt, aes_decrypt from lbryschema.address import public_key_to_address -from .lbrycrd import deserialize_xkey -from .lbrycrd import CKD_pub -log = logging.getLogger(__name__) +class KeyChain: - -def get_key_chain_from_xpub(xpub): - _, _, _, chain, key = deserialize_xkey(xpub) - return key, chain - - -class AddressSequence: - - def __init__(self, derived_keys, gap, age_checker, pub_key, chain_key): - self.gap = gap - self.is_old = age_checker - self.pub_key = pub_key - self.chain_key = chain_key - self.derived_keys = derived_keys + def __init__(self, parent_key, child_keys, gap): + self.parent_key = parent_key # type: PubKey + self.child_keys = child_keys + self.minimum_gap = gap self.addresses = [ - public_key_to_address(key.decode('hex')) - for key in derived_keys + public_key_to_address(key) + for key in child_keys ] - def generate_next_address(self): - new_key, _ = CKD_pub(self.pub_key, self.chain_key, len(self.derived_keys)) - address = public_key_to_address(new_key) - self.derived_keys.append(new_key.encode('hex')) - self.addresses.append(address) - return address - + @property def has_gap(self): - if len(self.addresses) < self.gap: + if len(self.addresses) < self.minimum_gap: return False - for address in self.addresses[-self.gap:]: - if self.is_old(address): + ledger = get_wallet_manager().ledger + for address in self.addresses[-self.minimum_gap:]: + if ledger.is_address_old(address): return False return True + def generate_next_address(self): + child_key = self.parent_key.child(len(self.child_keys)) + self.child_keys.append(child_key.pubkey_bytes) + self.addresses.append(child_key.address) + return child_key.address + def ensure_enough_addresses(self): starting_length = len(self.addresses) - while not self.has_gap(): + while not self.has_gap: self.generate_next_address() return self.addresses[starting_length:] class Account: - def __init__(self, data, receiving_gap, change_gap, age_checker): - self.xpub = data['xpub'] - master_key, master_chain = get_key_chain_from_xpub(data['xpub']) - self.receiving = AddressSequence( - data.get('receiving', []), receiving_gap, age_checker, - *CKD_pub(master_key, master_chain, 0) - ) - self.change = AddressSequence( - data.get('change', []), change_gap, age_checker, - *CKD_pub(master_key, master_chain, 1) - ) - self.is_old = age_checker + def __init__(self, seed, encrypted, private_key, public_key, **kwargs): + self.seed = seed + self.encrypted = encrypted + self.private_key = private_key # type: PrivateKey + self.public_key = public_key # type: PubKey + self.receiving_gap = kwargs.get('receiving_gap', 20) + self.receiving_keys = kwargs.get('receiving_keys') or \ + KeyChain(self.public_key.child(0), [], self.receiving_gap) + self.change_gap = kwargs.get('change_gap', 6) + self.change_keys = kwargs.get('change_keys') or \ + KeyChain(self.public_key.child(1), [], self.change_gap) + self.keychains = [ + self.receiving_keys, # child: 0 + self.change_keys # child: 1 + ] - def as_dict(self): + @classmethod + def generate(cls): + seed = Mnemonic().make_seed() + return cls.generate_from_seed(seed) + + @classmethod + def generate_from_seed(cls, seed): + private_key = cls.get_private_key_from_seed(seed) + return cls( + seed=seed, encrypted=False, + private_key=private_key, + public_key=private_key.public_key, + ) + + @classmethod + def from_json(cls, json_data): + data = json_data.copy() + if not data['encrypted']: + data['private_key'] = from_extended_key_string(data['private_key']) + data['public_key'] = from_extended_key_string(data['public_key']) + data['receiving_keys'] = KeyChain( + data['public_key'].child(0), + [unhexlify(k) for k in data['receiving_keys']], + data['receiving_gap'] + ) + data['change_keys'] = KeyChain( + data['public_key'].child(1), + [unhexlify(k) for k in data['change_keys']], + data['change_gap'] + ) + return cls(**data) + + def to_json(self): return { - 'receiving': self.receiving.derived_keys, - 'change': self.change.derived_keys, - 'xpub': self.xpub + 'seed': self.seed, + 'encrypted': self.encrypted, + 'private_key': self.private_key.extended_key_string(), + 'public_key': self.public_key.extended_key_string(), + 'receiving_keys': [hexlify(k) for k in self.receiving_keys.child_keys], + 'receiving_gap': self.receiving_gap, + 'change_keys': [hexlify(k) for k in self.change_keys.child_keys], + 'change_gap': self.change_gap } + def decrypt(self, password): + assert self.encrypted, "Key is not encrypted." + secret = double_sha256(password) + self.seed = aes_decrypt(secret, self.seed) + self.private_key = from_extended_key_string(aes_decrypt(secret, self.private_key)) + self.encrypted = False + + def encrypt(self, password): + assert not self.encrypted, "Key is already encrypted." + secret = double_sha256(password) + self.seed = aes_encrypt(secret, self.seed) + self.private_key = aes_encrypt(secret, self.private_key.extended_key_string()) + self.encrypted = True + + @staticmethod + def get_private_key_from_seed(seed): + return PrivateKey.from_seed(Mnemonic.mnemonic_to_seed(seed)) + @property - def sequences(self): - return self.receiving, self.change + def addresses(self): + return chain(self.receiving_keys.addresses, self.change_keys.addresses) + + def get_private_key_for_address(self, address): + assert not self.encrypted, "Cannot get private key on encrypted wallet account." + for a, keychain in enumerate(self.keychains): + for b, match in enumerate(keychain.addresses): + if address == match: + return self.private_key.child(a).child(b) + + def ensure_enough_addresses(self): + return [ + address + for keychain in self.keychains + for address in keychain.ensure_enough_addresses() + ] diff --git a/lbrynet/wallet/bip32.py b/lbrynet/wallet/bip32.py new file mode 100644 index 000000000..3cb5082c2 --- /dev/null +++ b/lbrynet/wallet/bip32.py @@ -0,0 +1,320 @@ +# Copyright (c) 2017, Neil Booth +# Copyright (c) 2018, LBRY Inc. +# +# All rights reserved. +# +# See the file "LICENCE" for information about the copyright +# and warranty status of this software. + +""" Logic for BIP32 Hierarchical Key Derivation. """ + +import struct +import hashlib +from binascii import unhexlify +from six import int2byte, byte2int + +import ecdsa +import ecdsa.ellipticcurve as EC +import ecdsa.numbertheory as NT + +from .hash import Base58, hmac_sha512, hash160, double_sha256, public_key_to_address +from .util import cachedproperty, bytes_to_int, int_to_bytes + + +class DerivationError(Exception): + """ Raised when an invalid derivation occurs. """ + + +class _KeyBase(object): + """ A BIP32 Key, public or private. """ + + CURVE = ecdsa.SECP256k1 + + def __init__(self, chain_code, n, depth, parent): + if not isinstance(chain_code, (bytes, bytearray)): + raise TypeError('chain code must be raw bytes') + if len(chain_code) != 32: + raise ValueError('invalid chain code') + if not 0 <= n < 1 << 32: + raise ValueError('invalid child number') + if not 0 <= depth < 256: + raise ValueError('invalid depth') + if parent is not None: + if not isinstance(parent, type(self)): + raise TypeError('parent key has bad type') + self.chain_code = chain_code + self.n = n + self.depth = depth + self.parent = parent + + def _hmac_sha512(self, msg): + """ Use SHA-512 to provide an HMAC, returned as a pair of 32-byte objects. """ + hmac = hmac_sha512(self.chain_code, msg) + return hmac[:32], hmac[32:] + + def _extended_key(self, ver_bytes, raw_serkey): + """ Return the 78-byte extended key given prefix version bytes and serialized key bytes. """ + if not isinstance(ver_bytes, (bytes, bytearray)): + raise TypeError('ver_bytes must be raw bytes') + if len(ver_bytes) != 4: + raise ValueError('ver_bytes must have length 4') + if not isinstance(raw_serkey, (bytes, bytearray)): + raise TypeError('raw_serkey must be raw bytes') + if len(raw_serkey) != 33: + raise ValueError('raw_serkey must have length 33') + + return (ver_bytes + int2byte(self.depth) + + self.parent_fingerprint() + struct.pack('>I', self.n) + + self.chain_code + raw_serkey) + + def fingerprint(self): + """ Return the key's fingerprint as 4 bytes. """ + return self.identifier()[:4] + + def parent_fingerprint(self): + """ Return the parent key's fingerprint as 4 bytes. """ + return self.parent.fingerprint() if self.parent else int2byte(0)*4 + + def extended_key_string(self): + """ Return an extended key as a base58 string. """ + return Base58.encode_check(self.extended_key()) + + +class PubKey(_KeyBase): + """ A BIP32 public key. """ + + def __init__(self, pubkey, chain_code, n, depth, parent=None): + super(PubKey, self).__init__(chain_code, n, depth, parent) + if isinstance(pubkey, ecdsa.VerifyingKey): + self.verifying_key = pubkey + else: + self.verifying_key = self._verifying_key_from_pubkey(pubkey) + + @classmethod + def _verifying_key_from_pubkey(cls, pubkey): + """ Converts a 33-byte compressed pubkey into an ecdsa.VerifyingKey object. """ + if not isinstance(pubkey, (bytes, bytearray)): + raise TypeError('pubkey must be raw bytes') + if len(pubkey) != 33: + raise ValueError('pubkey must be 33 bytes') + if byte2int(pubkey[0]) not in (2, 3): + raise ValueError('invalid pubkey prefix byte') + curve = cls.CURVE.curve + + is_odd = byte2int(pubkey[0]) == 3 + x = bytes_to_int(pubkey[1:]) + + # p is the finite field order + a, b, p = curve.a(), curve.b(), curve.p() + y2 = pow(x, 3, p) + b + assert a == 0 # Otherwise y2 += a * pow(x, 2, p) + y = NT.square_root_mod_prime(y2 % p, p) + if bool(y & 1) != is_odd: + y = p - y + point = EC.Point(curve, x, y) + + return ecdsa.VerifyingKey.from_public_point(point, curve=cls.CURVE) + + @cachedproperty + def pubkey_bytes(self): + """ Return the compressed public key as 33 bytes. """ + point = self.verifying_key.pubkey.point + prefix = int2byte(2 + (point.y() & 1)) + padded_bytes = _exponent_to_bytes(point.x()) + return prefix + padded_bytes + + @cachedproperty + def address(self): + """ The public key as a P2PKH address. """ + return public_key_to_address(self.pubkey_bytes, 'regtest') + + def ec_point(self): + return self.verifying_key.pubkey.point + + def child(self, n): + """ Return the derived child extended pubkey at index N. """ + if not 0 <= n < (1 << 31): + raise ValueError('invalid BIP32 public key child number') + + msg = self.pubkey_bytes + struct.pack('>I', n) + L, R = self._hmac_sha512(msg) + + curve = self.CURVE + L = bytes_to_int(L) + if L >= curve.order: + raise DerivationError + + point = curve.generator * L + self.ec_point() + if point == EC.INFINITY: + raise DerivationError + + verkey = ecdsa.VerifyingKey.from_public_point(point, curve=curve) + + return PubKey(verkey, R, n, self.depth + 1, self) + + def identifier(self): + """ Return the key's identifier as 20 bytes. """ + return hash160(self.pubkey_bytes) + + def extended_key(self): + """ Return a raw extended public key. """ + return self._extended_key(unhexlify("0488b21e"), self.pubkey_bytes) + + +class LowSValueSigningKey(ecdsa.SigningKey): + """ + Enforce low S values in signatures + BIP-0062: https://github.com/bitcoin/bips/blob/master/bip-0062.mediawiki#low-s-values-in-signatures + """ + + def sign_number(self, number, entropy=None, k=None): + order = self.privkey.order + r, s = ecdsa.SigningKey.sign_number(self, number, entropy, k) + if s > order / 2: + s = order - s + return r, s + + +class PrivateKey(_KeyBase): + """A BIP32 private key.""" + + HARDENED = 1 << 31 + + def __init__(self, privkey, chain_code, n, depth, parent=None): + super(PrivateKey, self).__init__(chain_code, n, depth, parent) + if isinstance(privkey, ecdsa.SigningKey): + self.signing_key = privkey + else: + self.signing_key = self._signing_key_from_privkey(privkey) + + @classmethod + def _signing_key_from_privkey(cls, private_key): + """ Converts a 32-byte private key into an ecdsa.SigningKey object. """ + exponent = cls._private_key_secret_exponent(private_key) + return LowSValueSigningKey.from_secret_exponent(exponent, curve=cls.CURVE) + + @classmethod + def _private_key_secret_exponent(cls, private_key): + """ Return the private key as a secret exponent if it is a valid private key. """ + if not isinstance(private_key, (bytes, bytearray)): + raise TypeError('private key must be raw bytes') + if len(private_key) != 32: + raise ValueError('private key must be 32 bytes') + exponent = bytes_to_int(private_key) + if not 1 <= exponent < cls.CURVE.order: + raise ValueError('private key represents an invalid exponent') + return exponent + + @classmethod + def from_seed(cls, seed): + # This hard-coded message string seems to be coin-independent... + hmac = hmac_sha512(b'Bitcoin seed', seed) + privkey, chain_code = hmac[:32], hmac[32:] + return cls(privkey, chain_code, 0, 0) + + @cachedproperty + def private_key_bytes(self): + """ Return the serialized private key (no leading zero byte). """ + return _exponent_to_bytes(self.secret_exponent()) + + @cachedproperty + def public_key(self): + """ Return the corresponding extended public key. """ + verifying_key = self.signing_key.get_verifying_key() + parent_pubkey = self.parent.public_key if self.parent else None + return PubKey(verifying_key, self.chain_code, self.n, self.depth, + parent_pubkey) + + def ec_point(self): + return self.public_key.ec_point() + + def secret_exponent(self): + """ Return the private key as a secret exponent. """ + return self.signing_key.privkey.secret_multiplier + + def wif(self): + """ Return the private key encoded in Wallet Import Format. """ + return b'\x1c' + self.private_key_bytes + b'\x01' + + def address(self): + """ The public key as a P2PKH address. """ + return self.public_key.address + + def child(self, n): + """ Return the derived child extended private key at index N.""" + if not 0 <= n < (1 << 32): + raise ValueError('invalid BIP32 private key child number') + + if n >= self.HARDENED: + serkey = b'\0' + self.private_key_bytes + else: + serkey = self.public_key.pubkey_bytes + + msg = serkey + struct.pack('>I', n) + L, R = self._hmac_sha512(msg) + + curve = self.CURVE + L = bytes_to_int(L) + exponent = (L + bytes_to_int(self.private_key_bytes)) % curve.order + if exponent == 0 or L >= curve.order: + raise DerivationError + + privkey = _exponent_to_bytes(exponent) + + return PrivateKey(privkey, R, n, self.depth + 1, self) + + def sign(self, data): + """ Produce a signature for piece of data by double hashing it and signing the hash. """ + key = self.signing_key + digest = double_sha256(data) + return key.sign_digest_deterministic(digest, hashlib.sha256, ecdsa.util.sigencode_der) + + def identifier(self): + """Return the key's identifier as 20 bytes.""" + return self.public_key.identifier() + + def extended_key(self): + """Return a raw extended private key.""" + return self._extended_key(unhexlify("0488ade4"), b'\0' + self.private_key_bytes) + + +def _exponent_to_bytes(exponent): + """Convert an exponent to 32 big-endian bytes""" + return (int2byte(0)*32 + int_to_bytes(exponent))[-32:] + + +def _from_extended_key(ekey): + """Return a PubKey or PrivateKey from an extended key raw bytes.""" + if not isinstance(ekey, (bytes, bytearray)): + raise TypeError('extended key must be raw bytes') + if len(ekey) != 78: + raise ValueError('extended key must have length 78') + + depth = byte2int(ekey[4]) + fingerprint = ekey[5:9] # Not used + n, = struct.unpack('>I', ekey[9:13]) + chain_code = ekey[13:45] + + if ekey[:4] == unhexlify("0488b21e"): + pubkey = ekey[45:] + key = PubKey(pubkey, chain_code, n, depth) + elif ekey[:4] == unhexlify("0488ade4"): + if ekey[45] is not int2byte(0): + raise ValueError('invalid extended private key prefix byte') + privkey = ekey[46:] + key = PrivateKey(privkey, chain_code, n, depth) + else: + raise ValueError('version bytes unrecognised') + + return key + + +def from_extended_key_string(ekey_str): + """Given an extended key string, such as + + xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd + 3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL + + return a PubKey or PrivateKey. + """ + return _from_extended_key(Base58.decode_check(ekey_str)) diff --git a/lbrynet/wallet/coinselection.py b/lbrynet/wallet/coinselection.py index 3e9080731..5637d434a 100644 --- a/lbrynet/wallet/coinselection.py +++ b/lbrynet/wallet/coinselection.py @@ -19,22 +19,12 @@ class CoinSelector: debug and print([c.effective_amount for c in self.coins]) def select(self): - if self.target > self.available: - return if not self.coins: return + if self.target > self.available: + return return self.branch_and_bound() or self.single_random_draw() - def single_random_draw(self): - self.random.shuffle(self.coins) - selection = [] - amount = 0 - for coin in self.coins: - selection.append(coin) - amount += coin.effective_amount - if amount >= self.target+self.cost_of_change: - return selection - def branch_and_bound(self): # see bitcoin implementation for more info: # https://github.com/bitcoin/bitcoin/blob/master/src/wallet/coinselection.cpp @@ -91,3 +81,13 @@ class CoinSelector: return [ self.coins[i] for i, include in enumerate(best_selection) if include ] + + def single_random_draw(self): + self.random.shuffle(self.coins) + selection = [] + amount = 0 + for coin in self.coins: + selection.append(coin) + amount += coin.effective_amount + if amount >= self.target+self.cost_of_change: + return selection diff --git a/lbrynet/wallet/constants.py b/lbrynet/wallet/constants.py index 55a166d0e..6abaed331 100644 --- a/lbrynet/wallet/constants.py +++ b/lbrynet/wallet/constants.py @@ -9,9 +9,11 @@ SEED_PREFIX = '01' # Electrum standard wallet SEED_PREFIX_2FA = '101' # extended seed for two-factor authentication -RECOMMENDED_FEE = 50000 +MAXIMUM_FEE_PER_BYTE = 50 +MAXIMUM_FEE_PER_NAME_CHAR = 200000 COINBASE_MATURITY = 100 -COIN = 100000000 +CENT = 1000000 +COIN = 100*CENT # supported types of transaction outputs TYPE_ADDRESS = 1 @@ -40,10 +42,13 @@ SERVER_RETRY_INTERVAL = 10 MAX_BATCH_QUERY_SIZE = 500 proxy_modes = ['socks4', 'socks5', 'http'] -# Main network and testnet3 definitions -# these values follow the parameters in lbrycrd/src/chainparams.cpp -blockchain_params = { - 'lbrycrd_main': { +# Chain Properties +# see: https://github.com/lbryio/lbrycrd/blob/master/src/chainparams.cpp +MAIN_CHAIN = 'main' +TESTNET_CHAIN = 'testnet' +REGTEST_CHAIN = 'regtest' +CHAINS = { + MAIN_CHAIN: { 'pubkey_address': 0, 'script_address': 5, 'pubkey_address_prefix': 85, @@ -53,7 +58,7 @@ blockchain_params = { 'genesis_bits': 0x1f00ffff, 'target_timespan': 150 }, - 'lbrycrd_testnet': { + TESTNET_CHAIN: { 'pubkey_address': 0, 'script_address': 5, 'pubkey_address_prefix': 111, @@ -63,7 +68,7 @@ blockchain_params = { 'genesis_bits': 0x1f00ffff, 'target_timespan': 150 }, - 'lbrycrd_regtest': { + REGTEST_CHAIN: { 'pubkey_address': 0, 'script_address': 5, 'pubkey_address_prefix': 111, diff --git a/lbrynet/wallet/enumeration.py b/lbrynet/wallet/enumeration.py deleted file mode 100644 index 497805a84..000000000 --- a/lbrynet/wallet/enumeration.py +++ /dev/null @@ -1,47 +0,0 @@ -import exceptions -import types - - -class EnumException(exceptions.Exception): - pass - - -class Enumeration(object): - """ - enum-like type - From the Python Cookbook, downloaded from http://code.activestate.com/recipes/67107/ - """ - - def __init__(self, name, enumList): - self.__doc__ = name - lookup = {} - reverseLookup = {} - i = 0 - uniqueNames = [] - uniqueValues = [] - for x in enumList: - if isinstance(x, types.TupleType): - x, i = x - if not isinstance(x, types.StringType): - raise EnumException, "enum name is not a string: " + x - if not isinstance(i, types.IntType): - raise EnumException, "enum value is not an integer: " + i - if x in uniqueNames: - raise EnumException, "enum name is not unique: " + x - if i in uniqueValues: - raise EnumException, "enum value is not unique for " + x - uniqueNames.append(x) - uniqueValues.append(i) - lookup[x] = i - reverseLookup[i] = x - i = i + 1 - self.lookup = lookup - self.reverseLookup = reverseLookup - - def __getattr__(self, attr): - if attr not in self.lookup: - raise AttributeError(attr) - return self.lookup[attr] - - def whatis(self, value): - return self.reverseLookup[value] diff --git a/lbrynet/wallet/hash.py b/lbrynet/wallet/hash.py new file mode 100644 index 000000000..5148f3d3a --- /dev/null +++ b/lbrynet/wallet/hash.py @@ -0,0 +1,183 @@ +# Copyright (c) 2016-2017, Neil Booth +# Copyright (c) 2018, LBRY Inc. +# +# All rights reserved. +# +# See the file "LICENCE" for information about the copyright +# and warranty status of this software. + +""" Cryptography hash functions and related classes. """ + +import six +import aes +import base64 +import hashlib +import hmac +import struct +from binascii import hexlify, unhexlify + +from .util import bytes_to_int, int_to_bytes +from .constants import CHAINS, MAIN_CHAIN + +_sha256 = hashlib.sha256 +_sha512 = hashlib.sha512 +_new_hash = hashlib.new +_new_hmac = hmac.new + + +def sha256(x): + """ Simple wrapper of hashlib sha256. """ + return _sha256(x).digest() + + +def sha512(x): + """ Simple wrapper of hashlib sha512. """ + return _sha512(x).digest() + + +def ripemd160(x): + """ Simple wrapper of hashlib ripemd160. """ + h = _new_hash('ripemd160') + h.update(x) + return h.digest() + + +def pow_hash(x): + r = sha512(double_sha256(x)) + r1 = ripemd160(r[:len(r) / 2]) + r2 = ripemd160(r[len(r) / 2:]) + r3 = double_sha256(r1 + r2) + return r3 + + +def double_sha256(x): + """ SHA-256 of SHA-256, as used extensively in bitcoin. """ + return sha256(sha256(x)) + + +def hmac_sha512(key, msg): + """ Use SHA-512 to provide an HMAC. """ + return _new_hmac(key, msg, _sha512).digest() + + +def hash160(x): + """ RIPEMD-160 of SHA-256. + Used to make bitcoin addresses from pubkeys. """ + return ripemd160(sha256(x)) + + +def hash_to_hex_str(x): + """ Convert a big-endian binary hash to displayed hex string. + Display form of a binary hash is reversed and converted to hex. """ + return hexlify(reversed(x)) + + +def hex_str_to_hash(x): + """ Convert a displayed hex string to a binary hash. """ + return reversed(unhexlify(x)) + + +def public_key_to_address(public_key, chain=MAIN_CHAIN): + return hash160_to_address(hash160(public_key), chain) + + +def hash160_to_address(h160, chain=MAIN_CHAIN): + prefix = CHAINS[chain]['pubkey_address_prefix'] + raw_address = six.int2byte(prefix) + h160 + return Base58.encode(raw_address + double_sha256(raw_address)[0:4]) + + +def address_to_hash_160(address): + bytes = Base58.decode(address) + prefix, pubkey_bytes, addr_checksum = bytes[0], bytes[1:21], bytes[21:] + return pubkey_bytes + + +def claim_id_hash(txid, n): + return hash160(txid + struct.pack('>I', n)) + + +def aes_encrypt(secret, value): + return base64.b64encode(aes.encryptData(secret, value.encode('utf8'))) + + +def aes_decrypt(secret, value): + return aes.decryptData(secret, base64.b64decode(value)).decode('utf8') + + +class Base58Error(Exception): + """ Exception used for Base58 errors. """ + + +class Base58(object): + """ Class providing base 58 functionality. """ + + chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + assert len(chars) == 58 + cmap = {c: n for n, c in enumerate(chars)} + + @staticmethod + def char_value(c): + val = Base58.cmap.get(c) + if val is None: + raise Base58Error('invalid base 58 character "{}"'.format(c)) + return val + + @staticmethod + def decode(txt): + """ Decodes txt into a big-endian bytearray. """ + if not isinstance(txt, str): + raise TypeError('a string is required') + + if not txt: + raise Base58Error('string cannot be empty') + + value = 0 + for c in txt: + value = value * 58 + Base58.char_value(c) + + result = int_to_bytes(value) + + # Prepend leading zero bytes if necessary + count = 0 + for c in txt: + if c != '1': + break + count += 1 + if count: + result = six.int2byte(0)*count + result + + return result + + @staticmethod + def encode(be_bytes): + """Converts a big-endian bytearray into a base58 string.""" + value = bytes_to_int(be_bytes) + + txt = '' + while value: + value, mod = divmod(value, 58) + txt += Base58.chars[mod] + + for byte in be_bytes: + if byte != 0: + break + txt += '1' + + return txt[::-1] + + @staticmethod + def decode_check(txt, hash_fn=double_sha256): + """ Decodes a Base58Check-encoded string to a payload. The version prefixes it. """ + be_bytes = Base58.decode(txt) + result, check = be_bytes[:-4], be_bytes[-4:] + if check != hash_fn(result)[:4]: + raise Base58Error('invalid base 58 checksum for {}'.format(txt)) + return result + + @staticmethod + def encode_check(payload, hash_fn=double_sha256): + """ Encodes a payload bytearray (which includes the version byte(s)) + into a Base58Check string.""" + be_bytes = payload + hash_fn(payload)[:4] + return Base58.encode(be_bytes) diff --git a/lbrynet/wallet/hashing.py b/lbrynet/wallet/hashing.py deleted file mode 100644 index ed50ee750..000000000 --- a/lbrynet/wallet/hashing.py +++ /dev/null @@ -1,50 +0,0 @@ -import hashlib -import hmac - - -def sha256(x): - return hashlib.sha256(x).digest() - - -def sha512(x): - return hashlib.sha512(x).digest() - - -def ripemd160(x): - h = hashlib.new('ripemd160') - h.update(x) - return h.digest() - - -def Hash(x): - if type(x) is unicode: - x = x.encode('utf-8') - return sha256(sha256(x)) - - -def PoWHash(x): - if type(x) is unicode: - x = x.encode('utf-8') - r = sha512(Hash(x)) - r1 = ripemd160(r[:len(r) / 2]) - r2 = ripemd160(r[len(r) / 2:]) - r3 = Hash(r1 + r2) - return r3 - - -def hash_encode(x): - return x[::-1].encode('hex') - - -def hash_decode(x): - return x.decode('hex')[::-1] - - -def hmac_sha_512(x, y): - return hmac.new(x, y, hashlib.sha512).digest() - - -def hash_160(public_key): - md = hashlib.new('ripemd160') - md.update(sha256(public_key)) - return md.digest() diff --git a/lbrynet/wallet/lbrycrd.py b/lbrynet/wallet/lbrycrd.py deleted file mode 100644 index d4bafe9bb..000000000 --- a/lbrynet/wallet/lbrycrd.py +++ /dev/null @@ -1,633 +0,0 @@ -import base64 -import hashlib -import hmac -import struct -import logging -import aes -import ecdsa -from ecdsa import numbertheory, util -from ecdsa.curves import SECP256k1 -from ecdsa.ecdsa import curve_secp256k1, generator_secp256k1 -from ecdsa.ellipticcurve import Point -from ecdsa.util import number_to_string, string_to_number - -from lbryschema.address import public_key_to_address -from lbryschema.schema import B58_CHARS -from lbryschema.base import b58encode_with_checksum, b58decode_strip_checksum - -from . import msqr -from .util import rev_hex, var_int, int_to_hex -from .hashing import Hash, sha256, hash_160 -from .errors import InvalidPassword, InvalidClaimId -from .constants import CLAIM_ID_SIZE - -log = logging.getLogger(__name__) - -# AES encryption -EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret, s)) -DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e)) - - -# get the claim id hash from txid bytes and int n -def claim_id_hash(txid, n): - return hash_160(txid + struct.pack('>I', n)) - - -# deocde a claim_id hex string -def decode_claim_id_hex(claim_id_hex): - claim_id = rev_hex(claim_id_hex).decode('hex') - if len(claim_id) != CLAIM_ID_SIZE: - raise InvalidClaimId() - return claim_id - - -# encode claim id bytes into hex string -def encode_claim_id_hex(claim_id): - return rev_hex(claim_id.encode('hex')) - - -def strip_PKCS7_padding(s): - """return s stripped of PKCS7 padding""" - if len(s) % 16 or not s: - raise ValueError("String of len %d can't be PCKS7-padded" % len(s)) - numpads = ord(s[-1]) - if numpads > 16: - raise ValueError("String ending with %r can't be PCKS7-padded" % s[-1]) - if s[-numpads:] != numpads * chr(numpads): - raise ValueError("Invalid PKCS7 padding") - return s[:-numpads] - - -# backport padding fix to AES module -aes.strip_PKCS7_padding = strip_PKCS7_padding - - -def aes_encrypt_with_iv(key, iv, data): - mode = aes.AESModeOfOperation.modeOfOperation["CBC"] - key = map(ord, key) - iv = map(ord, iv) - data = aes.append_PKCS7_padding(data) - keysize = len(key) - assert keysize in aes.AES.keySize.values(), 'invalid key size: %s' % keysize - moo = aes.AESModeOfOperation() - (mode, length, ciph) = moo.encrypt(data, mode, key, keysize, iv) - return ''.join(map(chr, ciph)) - - -def aes_decrypt_with_iv(key, iv, data): - mode = aes.AESModeOfOperation.modeOfOperation["CBC"] - key = map(ord, key) - iv = map(ord, iv) - keysize = len(key) - assert keysize in aes.AES.keySize.values(), 'invalid key size: %s' % keysize - data = map(ord, data) - moo = aes.AESModeOfOperation() - decr = moo.decrypt(data, None, mode, key, keysize, iv) - decr = strip_PKCS7_padding(decr) - return decr - - -def pw_encode(s, password): - if password: - secret = Hash(password) - return EncodeAES(secret, s.encode("utf8")) - else: - return s - - -def pw_decode(s, password): - if password is not None: - secret = Hash(password) - try: - d = DecodeAES(secret, s).decode("utf8") - except Exception: - raise InvalidPassword() - return d - else: - return s - - -def op_push(i): - if i < 0x4c: - return int_to_hex(i) - elif i < 0xff: - return '4c' + int_to_hex(i) - elif i < 0xffff: - return '4d' + int_to_hex(i, 2) - else: - return '4e' + int_to_hex(i, 4) - - -# pywallet openssl private key implementation - -def i2o_ECPublicKey(pubkey, compressed=False): - # public keys are 65 bytes long (520 bits) - # 0x04 + 32-byte X-coordinate + 32-byte Y-coordinate - # 0x00 = point at infinity, 0x02 and 0x03 = compressed, 0x04 = uncompressed - # compressed keys: <sign> <x> where <sign> is 0x02 if y is even and 0x03 if y is odd - if compressed: - if pubkey.point.y() & 1: - key = '03' + '%064x' % pubkey.point.x() - else: - key = '02' + '%064x' % pubkey.point.x() - else: - key = '04' + \ - '%064x' % pubkey.point.x() + \ - '%064x' % pubkey.point.y() - - return key.decode('hex') - - -# end pywallet openssl private key implementation -# functions from pywallet - - -def PrivKeyToSecret(privkey): - return privkey[9:9 + 32] - - -def SecretToASecret(secret, compressed=False, addrtype=0): - vchIn = chr((addrtype + 128) & 255) + secret - if compressed: - vchIn += '\01' - return b58encode_with_checksum(vchIn) - - -def ASecretToSecret(key, addrtype=0): - vch = b58decode_strip_checksum(key) - if vch and vch[0] == chr((addrtype + 128) & 255): - return vch[1:] - elif is_minikey(key): - return minikey_to_private_key(key) - else: - return False - - -def regenerate_key(sec): - b = ASecretToSecret(sec) - if not b: - return False - b = b[0:32] - return EC_KEY(b) - - -def GetPubKey(pubkey, compressed=False): - return i2o_ECPublicKey(pubkey, compressed) - - -def GetSecret(pkey): - return ('%064x' % pkey.secret).decode('hex') - - -def is_compressed(sec): - b = ASecretToSecret(sec) - return len(b) == 33 - - -def public_key_from_private_key(sec): - # rebuild public key from private key, compressed or uncompressed - pkey = regenerate_key(sec) - assert pkey - compressed = is_compressed(sec) - public_key = GetPubKey(pkey.pubkey, compressed) - return public_key.encode('hex') - - -def address_from_private_key(sec): - public_key = public_key_from_private_key(sec) - address = public_key_to_address(public_key.decode('hex')) - return address - - -def is_private_key(key): - try: - k = ASecretToSecret(key) - return k is not False - except: - return False - -# end pywallet functions - - -def is_minikey(text): - # Minikeys are typically 22 or 30 characters, but this routine - # permits any length of 20 or more provided the minikey is valid. - # A valid minikey must begin with an 'S', be in base58, and when - # suffixed with '?' have its SHA256 hash begin with a zero byte. - # They are widely used in Casascius physical bitoins. - return (len(text) >= 20 and text[0] == 'S' - and all(c in B58_CHARS for c in text) - and ord(sha256(text + '?')[0]) == 0) - - -def minikey_to_private_key(text): - return sha256(text) - - -def msg_magic(message): - varint = var_int(len(message)) - encoded_varint = "".join([chr(int(varint[i:i + 2], 16)) for i in xrange(0, len(varint), 2)]) - return "\x18Bitcoin Signed Message:\n" + encoded_varint + message - - -def verify_message(address, signature, message): - try: - EC_KEY.verify_message(address, signature, message) - return True - except Exception as e: - return False - - -def encrypt_message(message, pubkey): - return EC_KEY.encrypt_message(message, pubkey.decode('hex')) - - -def chunks(l, n): - return [l[i:i + n] for i in xrange(0, len(l), n)] - - -def ECC_YfromX(x, curved=curve_secp256k1, odd=True): - _p = curved.p() - _a = curved.a() - _b = curved.b() - for offset in range(128): - Mx = x + offset - My2 = pow(Mx, 3, _p) + _a * pow(Mx, 2, _p) + _b % _p - My = pow(My2, (_p + 1) / 4, _p) - - if curved.contains_point(Mx, My): - if odd == bool(My & 1): - return [My, offset] - return [_p - My, offset] - raise Exception('ECC_YfromX: No Y found') - - -def negative_point(P): - return Point(P.curve(), P.x(), -P.y(), P.order()) - - -def point_to_ser(P, comp=True): - if comp: - return (('%02x' % (2 + (P.y() & 1))) + ('%064x' % P.x())).decode('hex') - return ('04' + ('%064x' % P.x()) + ('%064x' % P.y())).decode('hex') - - -def ser_to_point(Aser): - curve = curve_secp256k1 - generator = generator_secp256k1 - _r = generator.order() - assert Aser[0] in ['\x02', '\x03', '\x04'] - if Aser[0] == '\x04': - return Point(curve, string_to_number(Aser[1:33]), string_to_number(Aser[33:]), _r) - Mx = string_to_number(Aser[1:]) - return Point(curve, Mx, ECC_YfromX(Mx, curve, Aser[0] == '\x03')[0], _r) - - -class MyVerifyingKey(ecdsa.VerifyingKey): - @classmethod - def from_signature(cls, sig, recid, h, curve): - """ See http://www.secg.org/download/aid-780/sec1-v2.pdf, chapter 4.1.6 """ - curveFp = curve.curve - G = curve.generator - order = G.order() - # extract r,s from signature - r, s = util.sigdecode_string(sig, order) - # 1.1 - x = r + (recid / 2) * order - # 1.3 - alpha = (x * x * x + curveFp.a() * x + curveFp.b()) % curveFp.p() - beta = msqr.modular_sqrt(alpha, curveFp.p()) - y = beta if (beta - recid) % 2 == 0 else curveFp.p() - beta - # 1.4 the constructor checks that nR is at infinity - R = Point(curveFp, x, y, order) - # 1.5 compute e from message: - e = string_to_number(h) - minus_e = -e % order - # 1.6 compute Q = r^-1 (sR - eG) - inv_r = numbertheory.inverse_mod(r, order) - Q = inv_r * (s * R + minus_e * G) - return cls.from_public_point(Q, curve) - - -class MySigningKey(ecdsa.SigningKey): - """Enforce low S values in signatures""" - - def sign_number(self, number, entropy=None, k=None): - curve = SECP256k1 - G = curve.generator - order = G.order() - r, s = ecdsa.SigningKey.sign_number(self, number, entropy, k) - if s > order / 2: - s = order - s - return r, s - - -class EC_KEY(object): - def __init__(self, k): - secret = string_to_number(k) - self.pubkey = ecdsa.ecdsa.Public_key(generator_secp256k1, generator_secp256k1 * secret) - self.privkey = ecdsa.ecdsa.Private_key(self.pubkey, secret) - self.secret = secret - - def get_public_key(self, compressed=True): - return point_to_ser(self.pubkey.point, compressed).encode('hex') - - def sign(self, msg_hash): - private_key = MySigningKey.from_secret_exponent(self.secret, curve=SECP256k1) - public_key = private_key.get_verifying_key() - signature = private_key.sign_digest_deterministic(msg_hash, hashfunc=hashlib.sha256, - sigencode=ecdsa.util.sigencode_string) - assert public_key.verify_digest(signature, msg_hash, sigdecode=ecdsa.util.sigdecode_string) - return signature - - def sign_message(self, message, compressed, address): - signature = self.sign(Hash(msg_magic(message))) - for i in range(4): - sig = chr(27 + i + (4 if compressed else 0)) + signature - try: - self.verify_message(address, sig, message) - return sig - except Exception: - log.exception("error: cannot sign message") - continue - raise Exception("error: cannot sign message") - - @classmethod - def verify_message(cls, address, sig, message): - if len(sig) != 65: - raise Exception("Wrong encoding") - nV = ord(sig[0]) - if nV < 27 or nV >= 35: - raise Exception("Bad encoding") - if nV >= 31: - compressed = True - nV -= 4 - else: - compressed = False - recid = nV - 27 - - h = Hash(msg_magic(message)) - public_key = MyVerifyingKey.from_signature(sig[1:], recid, h, curve=SECP256k1) - # check public key - public_key.verify_digest(sig[1:], h, sigdecode=ecdsa.util.sigdecode_string) - pubkey = point_to_ser(public_key.pubkey.point, compressed) - # check that we get the original signing address - addr = public_key_to_address(pubkey) - if address != addr: - raise Exception("Bad signature") - - # ECIES encryption/decryption methods; AES-128-CBC with PKCS7 is used as the cipher; - # hmac-sha256 is used as the mac - - @classmethod - def encrypt_message(cls, message, pubkey): - - pk = ser_to_point(pubkey) - if not ecdsa.ecdsa.point_is_valid(generator_secp256k1, pk.x(), pk.y()): - raise Exception('invalid pubkey') - - ephemeral_exponent = number_to_string(ecdsa.util.randrange(pow(2, 256)), - generator_secp256k1.order()) - ephemeral = EC_KEY(ephemeral_exponent) - ecdh_key = point_to_ser(pk * ephemeral.privkey.secret_multiplier) - key = hashlib.sha512(ecdh_key).digest() - iv, key_e, key_m = key[0:16], key[16:32], key[32:] - ciphertext = aes_encrypt_with_iv(key_e, iv, message) - ephemeral_pubkey = ephemeral.get_public_key(compressed=True).decode('hex') - encrypted = 'BIE1' + ephemeral_pubkey + ciphertext - mac = hmac.new(key_m, encrypted, hashlib.sha256).digest() - - return base64.b64encode(encrypted + mac) - - def decrypt_message(self, encrypted): - - encrypted = base64.b64decode(encrypted) - - if len(encrypted) < 85: - raise Exception('invalid ciphertext: length') - - magic = encrypted[:4] - ephemeral_pubkey = encrypted[4:37] - ciphertext = encrypted[37:-32] - mac = encrypted[-32:] - - if magic != 'BIE1': - raise Exception('invalid ciphertext: invalid magic bytes') - - try: - ephemeral_pubkey = ser_to_point(ephemeral_pubkey) - except AssertionError, e: - raise Exception('invalid ciphertext: invalid ephemeral pubkey') - - if not ecdsa.ecdsa.point_is_valid(generator_secp256k1, ephemeral_pubkey.x(), - ephemeral_pubkey.y()): - raise Exception('invalid ciphertext: invalid ephemeral pubkey') - - ecdh_key = point_to_ser(ephemeral_pubkey * self.privkey.secret_multiplier) - key = hashlib.sha512(ecdh_key).digest() - iv, key_e, key_m = key[0:16], key[16:32], key[32:] - if mac != hmac.new(key_m, encrypted[:-32], hashlib.sha256).digest(): - raise Exception('invalid ciphertext: invalid mac') - - return aes_decrypt_with_iv(key_e, iv, ciphertext) - - -# BIP32 - -def random_seed(n): - return "%032x" % ecdsa.util.randrange(pow(2, n)) - - -BIP32_PRIME = 0x80000000 - - -def get_pubkeys_from_secret(secret): - # public key - private_key = ecdsa.SigningKey.from_string(secret, curve=SECP256k1) - public_key = private_key.get_verifying_key() - K = public_key.to_string() - K_compressed = GetPubKey(public_key.pubkey, True) - return K, K_compressed - - -# Child private key derivation function (from master private key) -# k = master private key (32 bytes) -# c = master chain code (extra entropy for key derivation) (32 bytes) -# n = the index of the key we want to derive. (only 32 bits will be used) -# If n is negative (i.e. the 32nd bit is set), the resulting private key's -# corresponding public key can NOT be determined without the master private key. -# However, if n is positive, the resulting private key's corresponding -# public key can be determined without the master private key. -def CKD_priv(k, c, n): - is_prime = n & BIP32_PRIME - return _CKD_priv(k, c, rev_hex(int_to_hex(n, 4)).decode('hex'), is_prime) - - -def _CKD_priv(k, c, s, is_prime): - order = generator_secp256k1.order() - keypair = EC_KEY(k) - cK = GetPubKey(keypair.pubkey, True) - data = chr(0) + k + s if is_prime else cK + s - I = hmac.new(c, data, hashlib.sha512).digest() - k_n = number_to_string((string_to_number(I[0:32]) + string_to_number(k)) % order, order) - c_n = I[32:] - return k_n, c_n - - -# Child public key derivation function (from public key only) -# K = master public key -# c = master chain code -# n = index of key we want to derive -# This function allows us to find the nth public key, as long as n is -# non-negative. If n is negative, we need the master private key to find it. -def CKD_pub(cK, c, n): - if n & BIP32_PRIME: - raise Exception("CKD pub error") - return _CKD_pub(cK, c, rev_hex(int_to_hex(n, 4)).decode('hex')) - - -# helper function, callable with arbitrary string -def _CKD_pub(cK, c, s): - order = generator_secp256k1.order() - I = hmac.new(c, cK + s, hashlib.sha512).digest() - curve = SECP256k1 - pubkey_point = string_to_number(I[0:32]) * curve.generator + ser_to_point(cK) - public_key = ecdsa.VerifyingKey.from_public_point(pubkey_point, curve=SECP256k1) - c_n = I[32:] - cK_n = GetPubKey(public_key.pubkey, True) - return cK_n, c_n - - -BITCOIN_HEADER_PRIV = "0488ade4" -BITCOIN_HEADER_PUB = "0488b21e" - -TESTNET_HEADER_PRIV = "04358394" -TESTNET_HEADER_PUB = "043587cf" - -BITCOIN_HEADERS = (BITCOIN_HEADER_PUB, BITCOIN_HEADER_PRIV) -TESTNET_HEADERS = (TESTNET_HEADER_PUB, TESTNET_HEADER_PRIV) - - -def _get_headers(testnet): - """Returns the correct headers for either testnet or bitcoin, in the form - of a 2-tuple, like (public, private).""" - if testnet: - return TESTNET_HEADERS - else: - return BITCOIN_HEADERS - - -def deserialize_xkey(xkey): - xkey = b58decode_strip_checksum(xkey) - assert len(xkey) == 78 - - xkey_header = xkey[0:4].encode('hex') - # Determine if the key is a bitcoin key or a testnet key. - if xkey_header in TESTNET_HEADERS: - head = TESTNET_HEADER_PRIV - elif xkey_header in BITCOIN_HEADERS: - head = BITCOIN_HEADER_PRIV - else: - raise Exception("Unknown xkey header: '%s'" % xkey_header) - - depth = ord(xkey[4]) - fingerprint = xkey[5:9] - child_number = xkey[9:13] - c = xkey[13:13 + 32] - if xkey[0:4].encode('hex') == head: - K_or_k = xkey[13 + 33:] - else: - K_or_k = xkey[13 + 32:] - return depth, fingerprint, child_number, c, K_or_k - - -def get_xkey_name(xkey, testnet=False): - depth, fingerprint, child_number, c, K = deserialize_xkey(xkey) - n = int(child_number.encode('hex'), 16) - if n & BIP32_PRIME: - child_id = "%d'" % (n - BIP32_PRIME) - else: - child_id = "%d" % n - if depth == 0: - return '' - elif depth == 1: - return child_id - else: - raise BaseException("xpub depth error") - - -def xpub_from_xprv(xprv, testnet=False): - depth, fingerprint, child_number, c, k = deserialize_xkey(xprv) - K, cK = get_pubkeys_from_secret(k) - header_pub, _ = _get_headers(testnet) - xpub = header_pub.decode('hex') + chr(depth) + fingerprint + child_number + c + cK - return b58encode_with_checksum(xpub) - - -def bip32_root(seed, testnet=False): - header_pub, header_priv = _get_headers(testnet) - I = hmac.new("Bitcoin seed", seed, hashlib.sha512).digest() - master_k = I[0:32] - master_c = I[32:] - K, cK = get_pubkeys_from_secret(master_k) - xprv = (header_priv + "00" + "00000000" + "00000000").decode("hex") + master_c + chr( - 0) + master_k - xpub = (header_pub + "00" + "00000000" + "00000000").decode("hex") + master_c + cK - return b58encode_with_checksum(xprv), b58encode_with_checksum(xpub) - - -def xpub_from_pubkey(cK, testnet=False): - header_pub, header_priv = _get_headers(testnet) - assert cK[0] in ['\x02', '\x03'] - master_c = chr(0) * 32 - xpub = (header_pub + "00" + "00000000" + "00000000").decode("hex") + master_c + cK - return b58encode_with_checksum(xpub) - - -def bip32_private_derivation(xprv, branch, sequence, testnet=False): - assert sequence.startswith(branch) - if branch == sequence: - return xprv, xpub_from_xprv(xprv, testnet) - header_pub, header_priv = _get_headers(testnet) - depth, fingerprint, child_number, c, k = deserialize_xkey(xprv) - sequence = sequence[len(branch):] - for n in sequence.split('/'): - if n == '': - continue - i = int(n[:-1]) + BIP32_PRIME if n[-1] == "'" else int(n) - parent_k = k - k, c = CKD_priv(k, c, i) - depth += 1 - - _, parent_cK = get_pubkeys_from_secret(parent_k) - fingerprint = hash_160(parent_cK)[0:4] - child_number = ("%08X" % i).decode('hex') - K, cK = get_pubkeys_from_secret(k) - xprv = header_priv.decode('hex') + chr(depth) + fingerprint + child_number + c + chr(0) + k - xpub = header_pub.decode('hex') + chr(depth) + fingerprint + child_number + c + cK - return b58encode_with_checksum(xprv), b58encode_with_checksum(xpub) - - -def bip32_public_derivation(xpub, branch, sequence, testnet=False): - header_pub, _ = _get_headers(testnet) - depth, fingerprint, child_number, c, cK = deserialize_xkey(xpub) - assert sequence.startswith(branch) - sequence = sequence[len(branch):] - for n in sequence.split('/'): - if n == '': - continue - i = int(n) - parent_cK = cK - cK, c = CKD_pub(cK, c, i) - depth += 1 - - fingerprint = hash_160(parent_cK)[0:4] - child_number = ("%08X" % i).decode('hex') - xpub = header_pub.decode('hex') + chr(depth) + fingerprint + child_number + c + cK - return b58encode_with_checksum(xpub) - - -def bip32_private_key(sequence, k, chain): - for i in sequence: - k, chain = CKD_priv(k, chain, i) - return SecretToASecret(k, True) diff --git a/lbrynet/wallet/blockchain.py b/lbrynet/wallet/ledger.py similarity index 70% rename from lbrynet/wallet/blockchain.py rename to lbrynet/wallet/ledger.py index c08e0f44c..f26e38fa3 100644 --- a/lbrynet/wallet/blockchain.py +++ b/lbrynet/wallet/ledger.py @@ -1,36 +1,78 @@ import os import logging import hashlib +from binascii import hexlify +from operator import itemgetter from twisted.internet import threads, defer -from lbryum.util import hex_to_int, int_to_hex, rev_hex -from lbryum.hashing import hash_encode, Hash, PoWHash -from .stream import StreamController, execute_serially -from .constants import blockchain_params, HEADER_SIZE +from lbrynet.wallet.stream import StreamController, execute_serially +from lbrynet.wallet.transaction import Transaction +from lbrynet.wallet.constants import CHAINS, MAIN_CHAIN, REGTEST_CHAIN, HEADER_SIZE +from lbrynet.wallet.util import hex_to_int, int_to_hex, rev_hex, hash_encode +from lbrynet.wallet.hash import double_sha256, pow_hash log = logging.getLogger(__name__) -class Transaction: +class Address: - def __init__(self, tx_hash, raw, height): - self.hash = tx_hash - self.raw = raw - self.height = height + def __init__(self, address): + self.address = address + self.transactions = [] + + def add_transaction(self, transaction): + self.transactions.append(transaction) -class BlockchainTransactions: +class Ledger: - def __init__(self, history): + def __init__(self, config=None, db=None): + self.config = config or {} + self.db = db self.addresses = {} self.transactions = {} - for address, transactions in history.items(): - self.addresses[address] = [] - for txid, raw, height in transactions: - tx = Transaction(txid, raw, height) - self.addresses[address].append(tx) - self.transactions[txid] = tx + self.headers = BlockchainHeaders(self.headers_path, self.config.get('chain', MAIN_CHAIN)) + self._on_transaction_controller = StreamController() + self.on_transaction = self._on_transaction_controller.stream + + @property + def headers_path(self): + filename = 'blockchain_headers' + if self.config.get('chain', MAIN_CHAIN) != MAIN_CHAIN: + filename = '{}_headers'.format(self.config['chain']) + return os.path.join(self.config.get('wallet_path', ''), filename) + + @defer.inlineCallbacks + def load(self): + txs = yield self.db.get_transactions() + for tx_hash, raw, height in txs: + self.transactions[tx_hash] = Transaction(raw, height) + txios = yield self.db.get_transaction_inputs_and_outputs() + for tx_hash, address_hash, input_output, amount, height in txios: + tx = self.transactions[tx_hash] + address = self.addresses.get(address_hash) + if address is None: + address = self.addresses[address_hash] = Address(address_hash) + tx.add_txio(address, input_output, amount) + address.add_transaction(tx) + + def is_address_old(self, address, age_limit=2): + age = -1 + for tx in self.get_transactions(address, []): + if tx.height == 0: + tx_age = 0 + else: + tx_age = self.headers.height - tx.height + 1 + if tx_age > age: + age = tx_age + return age > age_limit + + def add_transaction(self, address, transaction): + self.transactions.setdefault(hexlify(transaction.id), transaction) + self.addresses.setdefault(address, []) + self.addresses[address].append(transaction) + self._on_transaction_controller.add(transaction) def has_address(self, address): return address in self.addresses @@ -52,28 +94,39 @@ class BlockchainTransactions: def has_transaction(self, tx_hash): return tx_hash in self.transactions - def add_transaction(self, address, transaction): - self.transactions.setdefault(transaction.hash, transaction) - self.addresses.setdefault(address, []) - self.addresses[address].append(transaction) + def get_least_used_address(self, addresses, max_transactions=100): + transaction_counts = [] + for address in addresses: + transactions = self.get_transactions(address, []) + tx_count = len(transactions) + if tx_count == 0: + return address + elif tx_count >= max_transactions: + continue + else: + transaction_counts.append((address, tx_count)) + if transaction_counts: + transaction_counts.sort(key=itemgetter(1)) + return transaction_counts[0] class BlockchainHeaders: - def __init__(self, path, chain='lbrycrd_main'): + def __init__(self, path, chain=MAIN_CHAIN): self.path = path self.chain = chain - self.max_target = blockchain_params[chain]['max_target'] - self.target_timespan = blockchain_params[chain]['target_timespan'] - self.genesis_bits = blockchain_params[chain]['genesis_bits'] + self.max_target = CHAINS[chain]['max_target'] + self.target_timespan = CHAINS[chain]['target_timespan'] + self.genesis_bits = CHAINS[chain]['genesis_bits'] self._on_change_controller = StreamController() self.on_changed = self._on_change_controller.stream self._size = None - if not os.path.exists(path): - with open(path, 'wb'): + def touch(self): + if not os.path.exists(self.path): + with open(self.path, 'wb'): pass @property @@ -175,12 +228,12 @@ class BlockchainHeaders: def _hash_header(self, header): if header is None: return '0' * 64 - return hash_encode(Hash(self._serialize(header).decode('hex'))) + return hash_encode(double_sha256(self._serialize(header).decode('hex'))) def _pow_hash_header(self, header): if header is None: return '0' * 64 - return hash_encode(PoWHash(self._serialize(header).decode('hex'))) + return hash_encode(pow_hash(self._serialize(header).decode('hex'))) def _calculate_lbry_next_work_required(self, height, first, last): """ See: lbrycrd/src/lbry.cpp """ @@ -189,7 +242,7 @@ class BlockchainHeaders: return self.genesis_bits, self.max_target # bits to target - if self.chain != 'lbrycrd_regtest': + if self.chain != REGTEST_CHAIN: bits = last['bits'] bitsN = (bits >> 24) & 0xff assert 0x03 <= bitsN <= 0x1f, \ diff --git a/lbrynet/wallet/manager.py b/lbrynet/wallet/manager.py index 6f52fbfb8..f396bb08a 100644 --- a/lbrynet/wallet/manager.py +++ b/lbrynet/wallet/manager.py @@ -1,80 +1,72 @@ -import os import logging +from binascii import unhexlify from operator import itemgetter - from twisted.internet import defer -import lbryschema - -from .protocol import Network -from .blockchain import BlockchainHeaders, Transaction -from .wallet import Wallet -from .stream import execute_serially +from lbrynet.wallet.wallet import Wallet +from lbrynet.wallet.ledger import Ledger +from lbrynet.wallet.protocol import Network +from lbrynet.wallet.transaction import Transaction +from lbrynet.wallet.stream import execute_serially +from lbrynet.wallet.constants import MAXIMUM_FEE_PER_BYTE, MAXIMUM_FEE_PER_NAME_CHAR log = logging.getLogger(__name__) class WalletManager: - def __init__(self, storage, config): - self.storage = storage - self.config = config - lbryschema.BLOCKCHAIN_NAME = config['chain'] - self.headers = BlockchainHeaders(self.headers_path, config['chain']) - self.wallet = Wallet(self.wallet_path, self.headers) - self.network = Network(config) + def __init__(self, config=None, wallet=None, ledger=None, network=None): + self.config = config or {} + self.ledger = ledger or Ledger(self.config) + self.wallet = wallet or Wallet() + self.wallets = [self.wallet] + self.network = network or Network(self.config) self.network.on_header.listen(self.process_header) self.network.on_status.listen(self.process_status) @property - def headers_path(self): - filename = 'blockchain_headers' - if self.config['chain'] != 'lbrycrd_main': - filename = '{}_headers'.format(self.config['chain'].split("_")[1]) - return os.path.join(self.config['wallet_path'], filename) + def fee_per_byte(self): + return self.config.get('fee_per_byte', MAXIMUM_FEE_PER_BYTE) @property - def wallet_path(self): - return os.path.join(self.config['wallet_path'], 'wallets', 'default_wallet') + def fee_per_name_char(self): + return self.config.get('fee_per_name_char', MAXIMUM_FEE_PER_NAME_CHAR) + + @property + def addresses_without_history(self): + for wallet in self.wallets: + for address in wallet.addresses: + if not self.ledger.has_address(address): + yield address def get_least_used_receiving_address(self, max_transactions=1000): return self._get_least_used_address( - self.wallet.receiving_addresses, - self.wallet.default_account.receiving, + self.wallet.default_account.receiving_keys.addresses, + self.wallet.default_account.receiving_keys, max_transactions ) def get_least_used_change_address(self, max_transactions=100): return self._get_least_used_address( - self.wallet.change_addresses, - self.wallet.default_account.change, + self.wallet.default_account.change_keys.addresses, + self.wallet.default_account.change_keys, max_transactions ) def _get_least_used_address(self, addresses, sequence, max_transactions): - transaction_counts = [] - for address in addresses: - transactions = self.wallet.history.get_transactions(address, []) - tx_count = len(transactions) - if tx_count == 0: - return address - elif tx_count >= max_transactions: - continue - else: - transaction_counts.append((address, tx_count)) - - if transaction_counts: - transaction_counts.sort(key=itemgetter(1)) - return transaction_counts[0] - + address = self.ledger.get_least_used_address(addresses, max_transactions) + if address: + return address address = sequence.generate_next_address() self.subscribe_history(address) return address @defer.inlineCallbacks def start(self): + first_connection = self.network.on_connected.first self.network.start() - yield self.network.on_connected.first + yield first_connection + self.ledger.headers.touch() yield self.update_headers() yield self.network.subscribe_headers() yield self.update_wallet() @@ -86,38 +78,34 @@ class WalletManager: @defer.inlineCallbacks def update_headers(self): while True: - height_sought = len(self.headers) + height_sought = len(self.ledger.headers) headers = yield self.network.get_headers(height_sought) log.info("received {} headers starting at {} height".format(headers['count'], height_sought)) if headers['count'] <= 0: break - yield self.headers.connect(height_sought, headers['hex'].decode('hex')) + yield self.ledger.headers.connect(height_sought, headers['hex'].decode('hex')) @defer.inlineCallbacks def process_header(self, response): header = response[0] if self.update_headers.is_running: return - if header['height'] == len(self.headers): + if header['height'] == len(self.ledger.headers): # New header from network directly connects after the last local header. - yield self.headers.connect(len(self.headers), header['hex'].decode('hex')) - elif header['height'] > len(self.headers): + yield self.ledger.headers.connect(len(self.ledger.headers), header['hex'].decode('hex')) + elif header['height'] > len(self.ledger.headers): # New header is several heights ahead of local, do download instead. yield self.update_headers() @execute_serially @defer.inlineCallbacks def update_wallet(self): - - if not self.wallet.exists: - self.wallet.create() - # Before subscribing, download history for any addresses that don't have any, # this avoids situation where we're getting status updates to addresses we know # need to update anyways. Continue to get history and create more addresses until # all missing addresses are created and history for them is fully restored. self.wallet.ensure_enough_addresses() - addresses = list(self.wallet.addresses_without_history) + addresses = list(self.addresses_without_history) while addresses: yield defer.gatherResults([ self.update_history(a) for a in addresses @@ -135,19 +123,19 @@ class WalletManager: def update_history(self, address): history = yield self.network.get_history(address) for hash in map(itemgetter('tx_hash'), history): - transaction = self.wallet.history.get_transaction(hash) + transaction = self.ledger.get_transaction(hash) if not transaction: raw = yield self.network.get_transaction(hash) - transaction = Transaction(hash, raw, None) - self.wallet.history.add_transaction(address, transaction) + transaction = Transaction(unhexlify(raw)) + self.ledger.add_transaction(address, transaction) @defer.inlineCallbacks def subscribe_history(self, address): status = yield self.network.subscribe_address(address) - if status != self.wallet.history.get_status(address): + if status != self.ledger.get_status(address): self.update_history(address) def process_status(self, response): address, status = response - if status != self.wallet.history.get_status(address): + if status != self.ledger.get_status(address): self.update_history(address) diff --git a/lbrynet/wallet/mnemonic.py b/lbrynet/wallet/mnemonic.py index 711b8ce23..e9eab6cea 100644 --- a/lbrynet/wallet/mnemonic.py +++ b/lbrynet/wallet/mnemonic.py @@ -5,14 +5,12 @@ import os import pkgutil import string import unicodedata -import logging import ecdsa import pbkdf2 from . import constants -from .hashing import hmac_sha_512 +from .hash import hmac_sha512 -log = logging.getLogger(__name__) # http://www.asahi-net.or.jp/~ax2s-kmtn/ref/unicode/e_asia.html CJK_INTERVALS = [ @@ -98,10 +96,9 @@ class Mnemonic: assert ' ' not in line if line: self.wordlist.append(line) - log.info("wordlist has %d words", len(self.wordlist)) @classmethod - def mnemonic_to_seed(cls, mnemonic, passphrase): + def mnemonic_to_seed(cls, mnemonic, passphrase=''): PBKDF2_ROUNDS = 2048 mnemonic = prepare_seed(mnemonic) return pbkdf2.PBKDF2(mnemonic, 'lbryum' + passphrase, iterations=PBKDF2_ROUNDS, @@ -137,7 +134,6 @@ class Mnemonic: k = len(prefix) * 4 # we add at least 16 bits n_added = max(16, k + num_bits - n) - log.info("make_seed %s adding %d bits", prefix, n_added) my_entropy = ecdsa.util.randrange(pow(2, n_added)) nonce = 0 while True: @@ -147,11 +143,10 @@ class Mnemonic: assert i == self.mnemonic_decode(seed) if is_new_seed(seed, prefix): break - log.info('%d words', len(seed.split())) return seed def is_new_seed(x, prefix=constants.SEED_PREFIX): x = prepare_seed(x) - s = hmac_sha_512("Seed version", x.encode('utf8')).encode('hex') + s = hmac_sha512("Seed version", x.encode('utf8')).encode('hex') return s.startswith(prefix) diff --git a/lbrynet/wallet/opcodes.py b/lbrynet/wallet/opcodes.py deleted file mode 100644 index 7527bc643..000000000 --- a/lbrynet/wallet/opcodes.py +++ /dev/null @@ -1,76 +0,0 @@ -import struct -from .enumeration import Enumeration - -opcodes = Enumeration("Opcodes", [ - ("OP_0", 0), ("OP_PUSHDATA1", 76), "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE", "OP_RESERVED", - "OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7", - "OP_8", "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16", - "OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF", "OP_ELSE", "OP_ENDIF", - "OP_VERIFY", - "OP_RETURN", "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP", "OP_2OVER", - "OP_2ROT", "OP_2SWAP", - "OP_IFDUP", "OP_DEPTH", "OP_DROP", "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL", - "OP_ROT", - "OP_SWAP", "OP_TUCK", "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE", "OP_INVERT", - "OP_AND", - "OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY", "OP_RESERVED1", "OP_RESERVED2", "OP_1ADD", - "OP_1SUB", "OP_2MUL", - "OP_2DIV", "OP_NEGATE", "OP_ABS", "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL", - "OP_DIV", - "OP_MOD", "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR", - "OP_NUMEQUAL", "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN", - "OP_GREATERTHAN", "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX", - "OP_WITHIN", "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160", - "OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG", - "OP_CHECKMULTISIGVERIFY", "OP_NOP1", "OP_NOP2", "OP_NOP3", "OP_NOP4", "OP_NOP5", - "OP_CLAIM_NAME", - "OP_SUPPORT_CLAIM", "OP_UPDATE_CLAIM", - ("OP_SINGLEBYTE_END", 0xF0), - ("OP_DOUBLEBYTE_BEGIN", 0xF000), - "OP_PUBKEY", "OP_PUBKEYHASH", - ("OP_INVALIDOPCODE", 0xFFFF), -]) - - -def script_GetOp(bytes): - i = 0 - while i < len(bytes): - vch = None - opcode = ord(bytes[i]) - i += 1 - if opcode >= opcodes.OP_SINGLEBYTE_END: - opcode <<= 8 - opcode |= ord(bytes[i]) - i += 1 - - if opcode <= opcodes.OP_PUSHDATA4: - nSize = opcode - if opcode == opcodes.OP_PUSHDATA1: - nSize = ord(bytes[i]) - i += 1 - elif opcode == opcodes.OP_PUSHDATA2: - (nSize,) = struct.unpack_from('<H', bytes, i) - i += 2 - elif opcode == opcodes.OP_PUSHDATA4: - (nSize,) = struct.unpack_from('<I', bytes, i) - i += 4 - vch = bytes[i:i + nSize] - i += nSize - - yield (opcode, vch, i) - - -def script_GetOpName(opcode): - return (opcodes.whatis(opcode)).replace("OP_", "") - - -def match_decoded(decoded, to_match): - if len(decoded) != len(to_match): - return False - for i, d in enumerate(decoded): - if to_match[i] == opcodes.OP_PUSHDATA4 and opcodes.OP_PUSHDATA4 >= d[0] > 0: - # Opcodes below OP_PUSHDATA4 all just push data onto stack, # and are equivalent. - continue - if to_match[i] != decoded[i][0]: - return False - return True diff --git a/lbrynet/wallet/protocol.py b/lbrynet/wallet/protocol.py index dc8cda58c..1ddf947ed 100644 --- a/lbrynet/wallet/protocol.py +++ b/lbrynet/wallet/protocol.py @@ -1,10 +1,9 @@ -import sys -import time +import six import json import socket import logging from itertools import cycle -from twisted.internet import defer, reactor, protocol, threads +from twisted.internet import defer, reactor, protocol from twisted.application.internet import ClientService, CancelledError from twisted.internet.endpoints import clientFromString from twisted.protocols.basic import LineOnlyReceiver @@ -16,6 +15,12 @@ from .stream import StreamController log = logging.getLogger() +def unicode2bytes(string): + if isinstance(string, six.text_type): + return string.encode('iso-8859-1') + return string + + class StratumClientProtocol(LineOnlyReceiver): delimiter = '\n' @@ -65,7 +70,14 @@ class StratumClientProtocol(LineOnlyReceiver): def lineReceived(self, line): try: - message = json.loads(line) + # `line` comes in as a byte string but `json.loads` automatically converts everything to + # unicode. For keys it's not a big deal but for values there is an expectation + # everywhere else in wallet code that most values are byte strings. + message = json.loads( + line, object_hook=lambda obj: { + k: unicode2bytes(v) for k, v in obj.items() + } + ) except (ValueError, TypeError): raise ProtocolException("Cannot decode message '{}'".format(line.strip())) @@ -137,7 +149,7 @@ class Network: @defer.inlineCallbacks def start(self): - for server in cycle(self.config.get('default_servers')): + for server in cycle(self.config['default_servers']): endpoint = clientFromString(reactor, 'tcp:{}:{}'.format(*server)) self.service = ClientService(endpoint, StratumClientFactory(self)) self.service.startService() diff --git a/lbrynet/wallet/script.py b/lbrynet/wallet/script.py index 89f1d21ce..f73037cdd 100644 --- a/lbrynet/wallet/script.py +++ b/lbrynet/wallet/script.py @@ -90,10 +90,30 @@ def read_small_integer(token): return (token - OP_1) + 1 -# tokens contain parsed values to be matched against opcodes -Token = namedtuple('Token', 'value') -DataToken = subclass_tuple('DataToken', Token) -SmallIntegerToken = subclass_tuple('SmallIntegerToken', Token) +class Token(namedtuple('Token', 'value')): + __slots__ = () + + def __repr__(self): + name = None + for var_name, var_value in globals().items(): + if var_name.startswith('OP_') and var_value == self.value: + name = var_name + break + return name or self.value + + +class DataToken(Token): + __slots__ = () + + def __repr__(self): + return '"{}"'.format(hexlify(self.value)) + + +class SmallIntegerToken(Token): + __slots__ = () + + def __repr__(self): + return 'SmallIntegerToken({})'.format(self.value) def token_producer(source): @@ -259,11 +279,13 @@ class Script(object): self.template = template self.values = values if source: - self._parse(template_hint) + self.parse(template_hint) elif template and values: - self.source = template.generate(values) - else: - raise ValueError("Either a valid 'source' or a 'template' and 'values' are required.") + self.generate() + + @property + def tokens(self): + return tokenize(BCDataStream(self.source)) @classmethod def from_source_with_template(cls, source, template): @@ -274,8 +296,8 @@ class Script(object): else: return cls(source, template_hint=template) - def _parse(self, template_hint=None): - tokens = tokenize(BCDataStream(self.source)) + def parse(self, template_hint=None): + tokens = self.tokens for template in chain((template_hint,), self.templates): if not template: continue @@ -287,12 +309,18 @@ class Script(object): continue raise ValueError('No matching templates for source: {}'.format(hexlify(self.source))) + def generate(self): + self.source = self.template.generate(self.values) + class InputScript(Script): + """ Input / redeem script templates (aka scriptSig) """ __slots__ = () - # input / redeem script templates (aka scriptSig) + REDEEM_PUBKEY = Template('pubkey', ( + PUSH_SINGLE('signature'), + )) REDEEM_PUBKEY_HASH = Template('pubkey_hash', ( PUSH_SINGLE('signature'), PUSH_SINGLE('pubkey') )) @@ -305,6 +333,7 @@ class InputScript(Script): )) templates = [ + REDEEM_PUBKEY, REDEEM_PUBKEY_HASH, REDEEM_SCRIPT_HASH, REDEEM_SCRIPT @@ -409,6 +438,14 @@ class OutputScript(Script): 'pubkey_hash': pubkey_hash }) + @property + def is_pay_pubkey_hash(self): + return self.template.name.endswith('pay_pubkey_hash') + + @property + def is_pay_script_hash(self): + return self.template.name.endswith('pay_script_hash') + @property def is_claim_name(self): return self.template.name.startswith('claim_name+') diff --git a/lbrynet/wallet/store.py b/lbrynet/wallet/store.py deleted file mode 100644 index 268a25f43..000000000 --- a/lbrynet/wallet/store.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import json - - -class JSONStore(dict): - - def __init__(self, config, name): - self.config = config - self.path = os.path.join(self.config.path, name) - self.load() - - def load(self): - try: - with open(self.path, 'r') as f: - self.update(json.loads(f.read())) - except: - pass - - def save(self): - with open(self.path, 'w') as f: - s = json.dumps(self, indent=4, sort_keys=True) - r = f.write(s) - - def __setitem__(self, key, value): - dict.__setitem__(self, key, value) - self.save() - - def pop(self, key): - if key in self.keys(): - dict.pop(self, key) - self.save() diff --git a/lbrynet/wallet/transaction.py b/lbrynet/wallet/transaction.py index 92b59f45d..5e17e6567 100644 --- a/lbrynet/wallet/transaction.py +++ b/lbrynet/wallet/transaction.py @@ -1,701 +1,346 @@ -import sys -import hashlib +import io +import six import logging -import ecdsa -from ecdsa.curves import SECP256k1 +from binascii import hexlify +from typing import List -from lbryschema.address import hash_160_bytes_to_address, public_key_to_address -from lbryschema.address import address_to_hash_160 +from lbrynet.wallet import get_wallet_manager +from lbrynet.wallet.bcd_data_stream import BCDataStream +from lbrynet.wallet.hash import sha256, hash160_to_address, claim_id_hash +from lbrynet.wallet.script import InputScript, OutputScript +from lbrynet.wallet.wallet import Wallet -from .constants import TYPE_SCRIPT, TYPE_PUBKEY, TYPE_UPDATE, TYPE_SUPPORT, TYPE_CLAIM -from .constants import TYPE_ADDRESS, NO_SIGNATURE -from .opcodes import opcodes, match_decoded, script_GetOp -from .bcd_data_stream import BCDataStream -from .hashing import Hash, hash_160, hash_encode -from .lbrycrd import op_push -from .lbrycrd import point_to_ser, MyVerifyingKey, MySigningKey -from .lbrycrd import regenerate_key, public_key_from_private_key -from .lbrycrd import encode_claim_id_hex, claim_id_hash -from .util import var_int, int_to_hex, parse_sig, rev_hex log = logging.getLogger() -def parse_xpub(x_pubkey): - if x_pubkey[0:2] in ['02', '03', '04']: - pubkey = x_pubkey - elif x_pubkey[0:2] == 'ff': - from lbryum.bip32 import BIP32_Account - xpub, s = BIP32_Account.parse_xpubkey(x_pubkey) - pubkey = BIP32_Account.derive_pubkey_from_xpub(xpub, s[0], s[1]) - elif x_pubkey[0:2] == 'fd': - addrtype = ord(x_pubkey[2:4].decode('hex')) - hash160 = x_pubkey[4:].decode('hex') - pubkey = None - address = hash_160_bytes_to_address(hash160, addrtype) - else: - raise BaseException("Cannnot parse pubkey") - if pubkey: - address = public_key_to_address(pubkey.decode('hex')) - return pubkey, address +NULL_HASH = '\x00'*32 -def parse_scriptSig(d, bytes): - try: - decoded = [x for x in script_GetOp(bytes)] - except Exception: - # coinbase transactions raise an exception - log.error("cannot find address in input script: {}".format(bytes.encode('hex'))) - return +class InputOutput(object): - # payto_pubkey - match = [opcodes.OP_PUSHDATA4] - if match_decoded(decoded, match): - sig = decoded[0][1].encode('hex') - d['address'] = "(pubkey)" - d['signatures'] = [sig] - d['num_sig'] = 1 - d['x_pubkeys'] = ["(pubkey)"] - d['pubkeys'] = ["(pubkey)"] - return + @property + def fee(self): + """ Fee based on size of the input / output. """ + return get_wallet_manager().fee_per_byte * self.size - # non-generated TxIn transactions push a signature - # (seventy-something bytes) and then their public key - # (65 bytes) onto the stack: - match = [opcodes.OP_PUSHDATA4, opcodes.OP_PUSHDATA4] - if match_decoded(decoded, match): - sig = decoded[0][1].encode('hex') - x_pubkey = decoded[1][1].encode('hex') - try: - signatures = parse_sig([sig]) - pubkey, address = parse_xpub(x_pubkey) - except: - import traceback - traceback.print_exc(file=sys.stdout) - log.error("cannot find address in input script: {}".format(bytes.encode('hex'))) - return - d['signatures'] = signatures - d['x_pubkeys'] = [x_pubkey] - d['num_sig'] = 1 - d['pubkeys'] = [pubkey] - d['address'] = address - return + @property + def size(self): + """ Size of this input / output in bytes. """ + stream = BCDataStream() + self.serialize_to(stream) + return len(stream.get_bytes()) - # p2sh transaction, m of n - match = [opcodes.OP_0] + [opcodes.OP_PUSHDATA4] * (len(decoded) - 1) - if not match_decoded(decoded, match): - log.error("cannot find address in input script: {}".format(bytes.encode('hex'))) - return - x_sig = [x[1].encode('hex') for x in decoded[1:-1]] - dec2 = [x for x in script_GetOp(decoded[-1][1])] - m = dec2[0][0] - opcodes.OP_1 + 1 - n = dec2[-2][0] - opcodes.OP_1 + 1 - op_m = opcodes.OP_1 + m - 1 - op_n = opcodes.OP_1 + n - 1 - match_multisig = [op_m] + [opcodes.OP_PUSHDATA4] * n + [op_n, opcodes.OP_CHECKMULTISIG] - if not match_decoded(dec2, match_multisig): - log.error("cannot find address in input script: {}".format(bytes.encode('hex'))) - return - x_pubkeys = map(lambda x: x[1].encode('hex'), dec2[1:-2]) - pubkeys = [parse_xpub(x)[0] for x in x_pubkeys] # xpub, addr = parse_xpub() - redeemScript = Transaction.multisig_script(pubkeys, m) - # write result in d - d['num_sig'] = m - d['signatures'] = parse_sig(x_sig) - d['x_pubkeys'] = x_pubkeys - d['pubkeys'] = pubkeys - d['redeemScript'] = redeemScript - d['address'] = hash_160_bytes_to_address(hash_160(redeemScript.decode('hex')), 5) + def serialize_to(self, stream): + raise NotImplemented -class NameClaim(object): - def __init__(self, name, value): - self.name = name - self.value = value +class Input(InputOutput): + NULL_SIGNATURE = '0'*72 + NULL_PUBLIC_KEY = '0'*33 -class ClaimUpdate(object): - def __init__(self, name, claim_id, value): - self.name = name - self.claim_id = claim_id - self.value = value - - -class ClaimSupport(object): - def __init__(self, name, claim_id): - self.name = name - self.claim_id = claim_id - - -def decode_claim_script(decoded_script): - if len(decoded_script) <= 6: - return False - op = 0 - claim_type = decoded_script[op][0] - if claim_type == opcodes.OP_UPDATE_CLAIM: - if len(decoded_script) <= 7: - return False - if claim_type not in [ - opcodes.OP_CLAIM_NAME, - opcodes.OP_SUPPORT_CLAIM, - opcodes.OP_UPDATE_CLAIM - ]: - return False - op += 1 - value = None - claim_id = None - claim = None - if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4: - return False - name = decoded_script[op][1] - op += 1 - if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4: - return False - if decoded_script[0][0] in [ - opcodes.OP_SUPPORT_CLAIM, - opcodes.OP_UPDATE_CLAIM - ]: - claim_id = decoded_script[op][1] - if len(claim_id) != 20: - return False - else: - value = decoded_script[op][1] - op += 1 - if decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM: - value = decoded_script[op][1] - op += 1 - if decoded_script[op][0] != opcodes.OP_2DROP: - return False - op += 1 - if decoded_script[op][0] != opcodes.OP_DROP and decoded_script[0][0] == opcodes.OP_CLAIM_NAME: - return False - elif decoded_script[op][0] != opcodes.OP_2DROP and decoded_script[0][0] == \ - opcodes.OP_UPDATE_CLAIM: - return False - op += 1 - if decoded_script[0][0] == opcodes.OP_CLAIM_NAME: - if name is None or value is None: - return False - claim = NameClaim(name, value) - elif decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM: - if name is None or value is None or claim_id is None: - return False - claim = ClaimUpdate(name, claim_id, value) - elif decoded_script[0][0] == opcodes.OP_SUPPORT_CLAIM: - if name is None or claim_id is None: - return False - claim = ClaimSupport(name, claim_id) - return claim, decoded_script[op:] - - -def get_address_from_output_script(script_bytes): - output_type = 0 - decoded = [x for x in script_GetOp(script_bytes)] - r = decode_claim_script(decoded) - claim_args = None - if r is not False: - claim_info, decoded = r - if isinstance(claim_info, NameClaim): - claim_args = (claim_info.name, claim_info.value) - output_type |= TYPE_CLAIM - elif isinstance(claim_info, ClaimSupport): - claim_args = (claim_info.name, claim_info.claim_id) - output_type |= TYPE_SUPPORT - elif isinstance(claim_info, ClaimUpdate): - claim_args = (claim_info.name, claim_info.claim_id, claim_info.value) - output_type |= TYPE_UPDATE - - # The Genesis Block, self-payments, and pay-by-IP-address payments look like: - # 65 BYTES:... CHECKSIG - match_pubkey = [opcodes.OP_PUSHDATA4, opcodes.OP_CHECKSIG] - - # Pay-by-Bitcoin-address TxOuts look like: - # DUP HASH160 20 BYTES:... EQUALVERIFY CHECKSIG - match_p2pkh = [opcodes.OP_DUP, opcodes.OP_HASH160, opcodes.OP_PUSHDATA4, opcodes.OP_EQUALVERIFY, - opcodes.OP_CHECKSIG] - - # p2sh - match_p2sh = [opcodes.OP_HASH160, opcodes.OP_PUSHDATA4, opcodes.OP_EQUAL] - - if match_decoded(decoded, match_pubkey): - output_val = decoded[0][1].encode('hex') - output_type |= TYPE_PUBKEY - elif match_decoded(decoded, match_p2pkh): - output_val = hash_160_bytes_to_address(decoded[2][1]) - output_type |= TYPE_ADDRESS - elif match_decoded(decoded, match_p2sh): - output_val = hash_160_bytes_to_address(decoded[1][1], 5) - output_type |= TYPE_ADDRESS - else: - output_val = bytes - output_type |= TYPE_SCRIPT - - if output_type & (TYPE_CLAIM | TYPE_SUPPORT | TYPE_UPDATE): - output_val = (claim_args, output_val) - - return output_type, output_val - - -def parse_input(vds): - d = {} - prevout_hash = hash_encode(vds.read_bytes(32)) - prevout_n = vds.read_uint32() - scriptSig = vds.read_bytes(vds.read_compact_size()) - d['scriptSig'] = scriptSig.encode('hex') - sequence = vds.read_uint32() - if prevout_hash == '00' * 32: - d['is_coinbase'] = True - else: - d['is_coinbase'] = False - d['prevout_hash'] = prevout_hash - d['prevout_n'] = prevout_n - d['sequence'] = sequence - d['pubkeys'] = [] - d['signatures'] = {} - d['address'] = None - if scriptSig: - parse_scriptSig(d, scriptSig) - return d - - -def parse_output(vds, i): - d = {} - d['value'] = vds.read_int64() - scriptPubKey = vds.read_bytes(vds.read_compact_size()) - d['type'], d['address'] = get_address_from_output_script(scriptPubKey) - d['scriptPubKey'] = scriptPubKey.encode('hex') - d['prevout_n'] = i - return d - - -def deserialize(raw): - vds = BCDataStream() - vds.write(raw.decode('hex')) - d = {} - start = vds.read_cursor - d['version'] = vds.read_int32() - n_vin = vds.read_compact_size() - d['inputs'] = list(parse_input(vds) for i in xrange(n_vin)) - n_vout = vds.read_compact_size() - d['outputs'] = list(parse_output(vds, i) for i in xrange(n_vout)) - d['lockTime'] = vds.read_uint32() - return d - - -def push_script(x): - return op_push(len(x) / 2) + x - - -class Transaction(object): - def __str__(self): - if self.raw is None: - self.raw = self.serialize() - return self.raw - - def __init__(self, raw): - if raw is None: - self.raw = None - elif type(raw) in [str, unicode]: - self.raw = raw.strip() if raw else None - elif type(raw) is dict: - self.raw = raw['hex'] + def __init__(self, output_or_txid_index, script, sequence=0xFFFFFFFF): + if isinstance(output_or_txid_index, Output): + self.output = output_or_txid_index # type: Output + self.output_txid = self.output.transaction.hash + self.output_index = self.output.index else: - raise BaseException("cannot initialize transaction", raw) - self._inputs = None - self._outputs = None + self.output = None # type: Output + self.output_txid, self.output_index = output_or_txid_index + self.sequence = sequence + self.is_coinbase = self.output_txid == NULL_HASH + self.coinbase = script if self.is_coinbase else None + self.script = script if not self.is_coinbase else None # type: InputScript - def update(self, raw): - self.raw = raw - self._inputs = None - self.deserialize() + def link_output(self, output): + assert self.output is None + assert self.output_txid == output.transaction.id + assert self.output_index == output.index + self.output = output - def inputs(self): - if self._inputs is None: - self.deserialize() - return self._inputs + @property + def amount(self): + """ Amount this input adds to the transaction. """ + if self.output is None: + raise ValueError('Cannot get input value without referenced output.') + return self.output.amount - def outputs(self): - if self._outputs is None: - self.deserialize() - return self._outputs + @property + def effective_amount(self): + """ Amount minus fee. """ + return self.amount - self.fee - def update_signatures(self, raw): - """Add new signatures to a transaction""" - d = deserialize(raw) - for i, txin in enumerate(self.inputs()): - sigs1 = txin.get('signatures') - sigs2 = d['inputs'][i].get('signatures') - for sig in sigs2: - if sig in sigs1: - continue - for_sig = Hash(self.tx_for_sig(i).decode('hex')) - # der to string - order = ecdsa.ecdsa.generator_secp256k1.order() - r, s = ecdsa.util.sigdecode_der(sig.decode('hex'), order) - sig_string = ecdsa.util.sigencode_string(r, s, order) - pubkeys = txin.get('pubkeys') - compressed = True - for recid in range(4): - public_key = MyVerifyingKey.from_signature(sig_string, recid, for_sig, - curve=SECP256k1) - pubkey = point_to_ser(public_key.pubkey.point, compressed).encode('hex') - if pubkey in pubkeys: - public_key.verify_digest(sig_string, for_sig, - sigdecode=ecdsa.util.sigdecode_string) - j = pubkeys.index(pubkey) - log.error("adding sig {} {} {} {}".format(i, j, pubkey, sig)) - self._inputs[i]['signatures'][j] = sig - self._inputs[i]['x_pubkeys'][j] = pubkey - break - # redo raw - self.raw = self.serialize() - - def deserialize(self): - if self.raw is None: - self.raw = self.serialize() - if self._inputs is not None: - return - d = deserialize(self.raw) - self._inputs = d['inputs'] - self._outputs = [(x['type'], x['address'], x['value']) for x in d['outputs']] - self.locktime = d['lockTime'] - return d + def __lt__(self, other): + return self.effective_amount < other.effective_amount @classmethod - def from_io(cls, inputs, outputs, locktime=0): - self = cls(None) - self._inputs = inputs - self._outputs = outputs - self.locktime = locktime + def deserialize_from(cls, stream): + txid = stream.read(32) + index = stream.read_uint32() + script = stream.read_string() + sequence = stream.read_uint32() + return cls( + (txid, index), + InputScript(script) if not txid == NULL_HASH else script, + sequence + ) + + def serialize_to(self, stream, alternate_script=None): + stream.write(self.output_txid) + stream.write_uint32(self.output_index) + if alternate_script is not None: + stream.write_string(alternate_script) + else: + if self.is_coinbase: + stream.write_string(self.coinbase) + else: + stream.write_string(self.script.source) + stream.write_uint32(self.sequence) + + def to_python_source(self): + return ( + u"InputScript(\n" + u" (output_txid=unhexlify('{}'), output_index={}),\n" + u" script=unhexlify('{}')\n" + u" # tokens: {}\n" + u")").format( + hexlify(self.output_txid), self.output_index, + hexlify(self.coinbase) if self.is_coinbase else hexlify(self.script.source), + repr(self.script.tokens) + ) + + +class Output(InputOutput): + + def __init__(self, transaction, index, amount, script): + self.transaction = transaction # type: Transaction + self.index = index # type: int + self.amount = amount # type: int + self.script = script # type: OutputScript + self._effective_amount = None # type: int + + def __lt__(self, other): + return self.effective_amount < other.effective_amount + + def _add_and_return(self): + self.transaction.add_outputs([self]) return self @classmethod - def multisig_script(cls, public_keys, m): - n = len(public_keys) - assert n <= 15 - assert m <= n - op_m = format(opcodes.OP_1 + m - 1, 'x') - op_n = format(opcodes.OP_1 + n - 1, 'x') - keylist = [op_push(len(k) / 2) + k for k in public_keys] - return op_m + ''.join(keylist) + op_n + 'ae' + def pay_pubkey_hash(cls, transaction, index, amount, pubkey_hash): + return cls( + transaction, index, amount, + OutputScript.pay_pubkey_hash(pubkey_hash) + )._add_and_return() @classmethod - def pay_script(cls, output_type, addr): - script = '' - if output_type & TYPE_CLAIM: - claim, addr = addr - claim_name, claim_value = claim - script += 'b5' # op_claim_name - script += push_script(claim_name.encode('hex')) - script += push_script(claim_value.encode('hex')) - script += '6d75' # op_2drop, op_drop - elif output_type & TYPE_SUPPORT: - claim, addr = addr - claim_name, claim_id = claim - script += 'b6' - script += push_script(claim_name.encode('hex')) - script += push_script(claim_id.encode('hex')) - script += '6d75' - elif output_type & TYPE_UPDATE: - claim, addr = addr - claim_name, claim_id, claim_value = claim - script += 'b7' - script += push_script(claim_name.encode('hex')) - script += push_script(claim_id.encode('hex')) - script += push_script(claim_value.encode('hex')) - script += '6d6d' + def pay_claim_name_pubkey_hash(cls, transaction, index, amount, claim_name, claim, pubkey_hash): + return cls( + transaction, index, amount, + OutputScript.pay_claim_name_pubkey_hash(claim_name, claim, pubkey_hash) + )._add_and_return() - if output_type & TYPE_SCRIPT: - script += addr.encode('hex') - elif output_type & TYPE_ADDRESS: # op_2drop, op_drop - addrtype, hash_160 = address_to_hash_160(addr) - if addrtype == 0: - script += '76a9' # op_dup, op_hash_160 - script += push_script(hash_160.encode('hex')) - script += '88ac' # op_equalverify, op_checksig - elif addrtype == 5: - script += 'a9' # op_hash_160 - script += push_script(hash_160.encode('hex')) - script += '87' # op_equal - else: - raise Exception("Unknown address type: %s" % addrtype) - else: - raise Exception("Unknown output type: %s" % output_type) - return script + def spend(self, signature=Input.NULL_SIGNATURE, pubkey=Input.NULL_PUBLIC_KEY): + """ Create the input to spend this output.""" + assert self.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.' + script = InputScript.redeem_pubkey_hash(signature, pubkey) + return Input(self, script) + + @property + def effective_amount(self): + """ Amount minus fees it would take to spend this output. """ + if self._effective_amount is None: + txi = self.spend() + self._effective_amount = txi.effective_amount + return self._effective_amount @classmethod - def input_script(cls, txin, i, for_sig): - # for_sig: - # -1 : do not sign, estimate length - # i>=0 : serialized tx for signing input i - # None : add all known signatures + def deserialize_from(cls, stream, transaction, index): + return cls( + transaction=transaction, + index=index, + amount=stream.read_uint64(), + script=OutputScript(stream.read_string()) + ) - p2sh = txin.get('redeemScript') is not None - num_sig = txin['num_sig'] if p2sh else 1 - address = txin['address'] + def serialize_to(self, stream): + stream.write_uint64(self.amount) + stream.write_string(self.script.source) - x_signatures = txin['signatures'] - signatures = filter(None, x_signatures) - is_complete = len(signatures) == num_sig + def to_python_source(self): + return ( + u"OutputScript(tx, index={}, amount={},\n" + u" script=unhexlify('{}')\n" + u" # tokens: {}\n" + u")").format( + self.index, self.amount, hexlify(self.script.source), repr(self.script.tokens)) - if for_sig in [-1, None]: - # if we have enough signatures, we use the actual pubkeys - # use extended pubkeys (with bip32 derivation) - if for_sig == -1: - # we assume that signature will be 0x48 bytes long - pubkeys = txin['pubkeys'] - sig_list = ["00" * 0x48] * num_sig - elif is_complete: - pubkeys = txin['pubkeys'] - sig_list = ((sig + '01') for sig in signatures) - else: - pubkeys = txin['x_pubkeys'] - sig_list = ((sig + '01') if sig else NO_SIGNATURE for sig in x_signatures) - script = ''.join(push_script(x) for x in sig_list) - if not p2sh: - x_pubkey = pubkeys[0] - if x_pubkey is None: - addrtype, h160 = address_to_hash_160(txin['address']) - x_pubkey = 'fd' + (chr(addrtype) + h160).encode('hex') - script += push_script(x_pubkey) - else: - script = '00' + script # put op_0 in front of script - redeem_script = cls.multisig_script(pubkeys, num_sig) - script += push_script(redeem_script) - elif for_sig == i: - script_type = TYPE_ADDRESS - if 'is_claim' in txin and txin['is_claim']: - script_type |= TYPE_CLAIM - address = ((txin['claim_name'], txin['claim_value']), address) - elif 'is_support' in txin and txin['is_support']: - script_type |= TYPE_SUPPORT - address = ((txin['claim_name'], txin['claim_id']), address) - elif 'is_update' in txin and txin['is_update']: - script_type |= TYPE_UPDATE - address = ((txin['claim_name'], txin['claim_id'], txin['claim_value']), address) - script = txin['redeemScript'] if p2sh else cls.pay_script(script_type, address) - else: - script = '' +class Transaction: - return script + def __init__(self, raw=None, version=1, locktime=0, height=None, is_saved=False): + self._raw = raw + self._hash = None + self._id = None + self.version = version # type: int + self.locktime = locktime # type: int + self.height = height # type: int + self.inputs = [] # type: List[Input] + self.outputs = [] # type: List[Output] + self.is_saved = is_saved # type: bool + if raw is not None: + self._deserialize() - @classmethod - def serialize_input(cls, txin, i, for_sig): - # Prev hash and index - s = txin['prevout_hash'].decode('hex')[::-1].encode('hex') - s += int_to_hex(txin['prevout_n'], 4) - # Script length, script, sequence - script = cls.input_script(txin, i, for_sig) - s += var_int(len(script) / 2) - s += script - s += "ffffffff" - return s - - def BIP_LI01_sort(self): - # See https://github.com/kristovatlas/rfc/blob/master/bips/bip-li01.mediawiki - self._inputs.sort(key=lambda i: (i['prevout_hash'], i['prevout_n'])) - self._outputs.sort(key=lambda o: (o[2], self.pay_script(o[0], o[1]))) - - def serialize(self, for_sig=None): - inputs = self.inputs() - outputs = self.outputs() - s = int_to_hex(1, 4) # version - s += var_int(len(inputs)) # number of inputs - for i, txin in enumerate(inputs): - s += self.serialize_input(txin, i, for_sig) - s += var_int(len(outputs)) # number of outputs - for output in outputs: - output_type, addr, amount = output - s += int_to_hex(amount, 8) # amount - script = self.pay_script(output_type, addr) - s += var_int(len(script) / 2) # script length - s += script # script - s += int_to_hex(0, 4) # lock time - if for_sig is not None and for_sig != -1: - s += int_to_hex(1, 4) # hash type - return s - - def tx_for_sig(self, i): - return self.serialize(for_sig=i) + @property + def id(self): + if self._id is None: + self._id = self.hash[::-1] + return self._id + @property def hash(self): - return Hash(self.raw.decode('hex'))[::-1].encode('hex') + if self._hash is None: + self._hash = sha256(sha256(self.raw)) + return self._hash - def get_claim_id(self, nout): - if nout < 0: - raise IndexError - if not self._outputs[nout][0] & TYPE_CLAIM: - raise ValueError - tx_hash = rev_hex(self.hash()).decode('hex') - return encode_claim_id_hex(claim_id_hash(tx_hash, nout)) + @property + def raw(self): + if self._raw is None: + self._raw = self._serialize() + return self._raw - def add_inputs(self, inputs): - self._inputs.extend(inputs) - self.raw = None + def _reset(self): + self._raw = None + self._hash = None + self._id = None - def add_outputs(self, outputs): - self._outputs.extend(outputs) - self.raw = None - - def input_value(self): - return sum(x['value'] for x in self.inputs()) - - def output_value(self): - return sum(val for tp, addr, val in self.outputs()) - - def get_fee(self): - return self.input_value() - self.output_value() - - def is_final(self): - return not any([x.get('sequence') < 0xffffffff - 1 for x in self.inputs()]) - - @classmethod - def fee_for_size(cls, relay_fee, fee_per_kb, size): - '''Given a fee per kB in satoshis, and a tx size in bytes, - returns the transaction fee.''' - fee = int(fee_per_kb * size / 1000.) - if fee < relay_fee: - fee = relay_fee - return fee - - def estimated_size(self): - '''Return an estimated tx size in bytes.''' - return len(self.serialize(-1)) / 2 # ASCII hex string - - @classmethod - def estimated_input_size(cls, txin): - '''Return an estimated of serialized input size in bytes.''' - return len(cls.serialize_input(txin, -1, -1)) / 2 - - def estimated_fee(self, relay_fee, fee_per_kb): - '''Return an estimated fee given a fee per kB in satoshis.''' - return self.fee_for_size(relay_fee, fee_per_kb, self.estimated_size()) - - def signature_count(self): - r = 0 - s = 0 - for txin in self.inputs(): - if txin.get('is_coinbase'): - continue - signatures = filter(None, txin.get('signatures', [])) - s += len(signatures) - r += txin.get('num_sig', -1) - return s, r + def get_claim_id(self, output_index): + script = self.outputs[output_index] + assert script.script.is_claim_name(), 'Not a name claim.' + return claim_id_hash(self.hash, output_index) + @property def is_complete(self): s, r = self.signature_count() return r == s - def inputs_without_script(self): - out = set() - for i, txin in enumerate(self.inputs()): - if txin.get('scriptSig') == '': - out.add(i) - return out + @property + def fee(self): + """ Fee that will actually be paid.""" + return self.input_sum - self.output_sum - def inputs_to_sign(self): - out = set() - for txin in self.inputs(): - num_sig = txin.get('num_sig') - if num_sig is None: - continue - x_signatures = txin['signatures'] - signatures = filter(None, x_signatures) - if len(signatures) == num_sig: - # input is complete - continue - for k, x_pubkey in enumerate(txin['x_pubkeys']): - if x_signatures[k] is not None: - # this pubkey already signed - continue - out.add(x_pubkey) - return out + @property + def size(self): + """ Size in bytes of the entire transaction. """ + return len(self.raw) - def sign(self, keypairs): - for i, txin in enumerate(self.inputs()): - num = txin['num_sig'] - for x_pubkey in txin['x_pubkeys']: - signatures = filter(None, txin['signatures']) - if len(signatures) == num: - # txin is complete - break - if x_pubkey in keypairs.keys(): - log.debug("adding signature for %s", x_pubkey) - # add pubkey to txin - txin = self._inputs[i] - x_pubkeys = txin['x_pubkeys'] - ii = x_pubkeys.index(x_pubkey) - sec = keypairs[x_pubkey] - pubkey = public_key_from_private_key(sec) - txin['x_pubkeys'][ii] = pubkey - txin['pubkeys'][ii] = pubkey - self._inputs[i] = txin - # add signature - for_sig = Hash(self.tx_for_sig(i).decode('hex')) - pkey = regenerate_key(sec) - secexp = pkey.secret - private_key = MySigningKey.from_secret_exponent(secexp, curve=SECP256k1) - public_key = private_key.get_verifying_key() - sig = private_key.sign_digest_deterministic(for_sig, hashfunc=hashlib.sha256, - sigencode=ecdsa.util.sigencode_der) - assert public_key.verify_digest(sig, for_sig, - sigdecode=ecdsa.util.sigdecode_der) - txin['signatures'][ii] = sig.encode('hex') - self._inputs[i] = txin - log.debug("is_complete: %s", self.is_complete()) - self.raw = self.serialize() + @property + def base_size(self): + """ Size in bytes of transaction meta data and all outputs; without inputs. """ + return len(self._serialize(with_inputs=False)) - def get_outputs(self): - """convert pubkeys to addresses""" - o = [] - for type, x, v in self.outputs(): - if type & (TYPE_CLAIM | TYPE_UPDATE | TYPE_SUPPORT): - x = x[1] - if type & TYPE_ADDRESS: - addr = x - elif type & TYPE_PUBKEY: - addr = public_key_to_address(x.decode('hex')) + @property + def base_fee(self): + """ Fee for the transaction header and all outputs; without inputs. """ + byte_fee = get_wallet_manager().fee_per_byte * self.base_size + return max(byte_fee, self.claim_name_fee) + + @property + def claim_name_fee(self): + char_fee = get_wallet_manager().fee_per_name_char + fee = 0 + for output in self.outputs: + if output.script.is_claim_name: + fee += len(output.script.values['claim_name']) * char_fee + return fee + + def _serialize(self, with_inputs=True): + stream = BCDataStream() + stream.write_uint32(self.version) + if with_inputs: + stream.write_compact_size(len(self.inputs)) + for txin in self.inputs: + txin.serialize_to(stream) + stream.write_compact_size(len(self.outputs)) + for txout in self.outputs: + txout.serialize_to(stream) + stream.write_uint32(self.locktime) + return stream.get_bytes() + + def _serialize_for_signature(self, signing_input): + stream = BCDataStream() + stream.write_uint32(self.version) + stream.write_compact_size(len(self.inputs)) + for i, txin in enumerate(self.inputs): + if signing_input == i: + txin.serialize_to(stream, txin.output.script.source) else: - addr = 'SCRIPT ' + x.encode('hex') - o.append((addr, v)) # consider using yield (addr, v) - return o + txin.serialize_to(stream, b'') + stream.write_compact_size(len(self.outputs)) + for txout in self.outputs: + txout.serialize_to(stream) + stream.write_uint32(self.locktime) + stream.write_uint32(1) # signature hash type: SIGHASH_ALL + return stream.get_bytes() - def get_output_addresses(self): - return [addr for addr, val in self.get_outputs()] + def _deserialize(self): + if self._raw is not None: + stream = BCDataStream(self._raw) + self.version = stream.read_uint32() + input_count = stream.read_compact_size() + self.inputs = [Input.deserialize_from(stream) for _ in range(input_count)] + output_count = stream.read_compact_size() + self.outputs = [Output.deserialize_from(stream, self, i) for i in range(output_count)] + self.locktime = stream.read_uint32() - def has_address(self, addr): - return (addr in self.get_output_addresses()) or ( - addr in (tx.get("address") for tx in self.inputs())) + def add_inputs(self, inputs): + self.inputs.extend(inputs) + self._reset() - def as_dict(self): - if self.raw is None: - self.raw = self.serialize() - self.deserialize() - out = { - 'hex': self.raw, - 'complete': self.is_complete() - } - return out + def add_outputs(self, outputs): + self.outputs.extend(outputs) + self._reset() - def requires_fee(self, wallet): - # see https://en.bitcoin.it/wiki/Transaction_fees - # - # size must be smaller than 1 kbyte for free tx - size = len(self.serialize(-1)) / 2 - if size >= 10000: - return True - # all outputs must be 0.01 BTC or larger for free tx - for addr, value in self.get_outputs(): - if value < 1000000: - return True - # priority must be large enough for free tx - threshold = 57600000 - weight = 0 - for txin in self.inputs(): - age = wallet.get_confirmations(txin["prevout_hash"])[0] - weight += txin["value"] * age - priority = weight / size - log.error("{} {}".format(priority, threshold)) + def sign(self, wallet): # type: (Wallet) -> bool + for i, txi in enumerate(self.inputs): + txo_script = txi.output.script + if txo_script.is_pay_pubkey_hash: + address = hash160_to_address(txo_script.values['pubkey_hash'], wallet.chain) + private_key = wallet.get_private_key_for_address(address) + tx = self._serialize_for_signature(i) + txi.script.values['signature'] = private_key.sign(tx)+six.int2byte(1) + txi.script.values['pubkey'] = private_key.public_key.pubkey_bytes + txi.script.generate() + self._reset() + return True - return priority < threshold + def sort(self): + # See https://github.com/kristovatlas/rfc/blob/master/bips/bip-li01.mediawiki + self.inputs.sort(key=lambda i: (i['prevout_hash'], i['prevout_n'])) + self.outputs.sort(key=lambda o: (o[2], pay_script(o[0], o[1]))) + + @property + def input_sum(self): + return sum(i.amount for i in self.inputs) + + @property + def output_sum(self): + return sum(o.amount for o in self.outputs) + + def to_python_source(self): + s = io.StringIO() + s.write(u'tx = Transaction(version={}, locktime={}, height={})\n'.format( + self.version, self.locktime, self.height + )) + for txi in self.inputs: + s.write(u'tx.add_input(') + s.write(txi.to_python_source()) + s.write(u')\n') + for txo in self.outputs: + s.write(u'tx.add_output(') + s.write(txo.to_python_source()) + s.write(u')\n') + s.write(u'# tx.id: unhexlify("{}")\n'.format(hexlify(self.id))) + s.write(u'# tx.raw: unhexlify("{}")\n'.format(hexlify(self.raw))) + return s.getvalue() diff --git a/lbrynet/wallet/util.py b/lbrynet/wallet/util.py index 0d0257f45..dcf5ee4f6 100644 --- a/lbrynet/wallet/util.py +++ b/lbrynet/wallet/util.py @@ -1,70 +1,32 @@ -import logging -import os -import re -from decimal import Decimal -import json -from .constants import NO_SIGNATURE - -log = logging.getLogger(__name__) +from binascii import unhexlify, hexlify def subclass_tuple(name, base): return type(name, (base,), {'__slots__': ()}) -def normalize_version(v): - return [int(x) for x in re.sub(r'(\.0+)*$', '', v).split(".")] +class cachedproperty(object): + + def __init__(self, f): + self.f = f + + def __get__(self, obj, type): + obj = obj or type + value = self.f(obj) + setattr(obj, self.f.__name__, value) + return value -def json_decode(x): - try: - return json.loads(x, parse_float=Decimal) - except: - return x +def bytes_to_int(be_bytes): + """ Interprets a big-endian sequence of bytes as an integer. """ + return int(hexlify(be_bytes), 16) -def user_dir(): - if "HOME" in os.environ: - return os.path.join(os.environ["HOME"], ".lbryum") - elif "APPDATA" in os.environ: - return os.path.join(os.environ["APPDATA"], "LBRYum") - elif "LOCALAPPDATA" in os.environ: - return os.path.join(os.environ["LOCALAPPDATA"], "LBRYum") - elif 'ANDROID_DATA' in os.environ: - try: - import jnius - env = jnius.autoclass('android.os.Environment') - _dir = env.getExternalStorageDirectory().getPath() - return _dir + '/lbryum/' - except ImportError: - pass - return "/sdcard/lbryum/" - else: - # raise Exception("No home directory found in environment variables.") - return - - -def format_satoshis(x, is_diff=False, num_zeros=0, decimal_point=8, whitespaces=False): - from locale import localeconv - if x is None: - return 'unknown' - x = int(x) # Some callers pass Decimal - scale_factor = pow(10, decimal_point) - integer_part = "{:n}".format(int(abs(x) / scale_factor)) - if x < 0: - integer_part = '-' + integer_part - elif is_diff: - integer_part = '+' + integer_part - dp = localeconv()['decimal_point'] - fract_part = ("{:0" + str(decimal_point) + "}").format(abs(x) % scale_factor) - fract_part = fract_part.rstrip('0') - if len(fract_part) < num_zeros: - fract_part += "0" * (num_zeros - len(fract_part)) - result = integer_part + dp + fract_part - if whitespaces: - result += " " * (decimal_point - len(fract_part)) - result = " " * (15 - len(result)) + result - return result.decode('utf8') +def int_to_bytes(value): + """ Converts an integer to a big-endian sequence of bytes. """ + length = (value.bit_length() + 7) // 8 + h = '%x' % value + return unhexlify(('0' * (len(h) % 2) + h).zfill(length * 2)) def rev_hex(s): @@ -81,41 +43,5 @@ def hex_to_int(s): return int('0x' + s[::-1].encode('hex'), 16) -def var_int(i): - # https://en.bitcoin.it/wiki/Protocol_specification#Variable_length_integer - if i < 0xfd: - return int_to_hex(i) - elif i <= 0xffff: - return "fd" + int_to_hex(i, 2) - elif i <= 0xffffffff: - return "fe" + int_to_hex(i, 4) - else: - return "ff" + int_to_hex(i, 8) - - -# This function comes from bitcointools, bct-LICENSE.txt. -def long_hex(bytes): - return bytes.encode('hex_codec') - - -# This function comes from bitcointools, bct-LICENSE.txt. -def short_hex(bytes): - t = bytes.encode('hex_codec') - if len(t) < 11: - return t - return t[0:4] + "..." + t[-4:] - - -def parse_sig(x_sig): - s = [] - for sig in x_sig: - if sig[-2:] == '01': - s.append(sig[:-2]) - else: - assert sig == NO_SIGNATURE - s.append(None) - return s - - -def is_extended_pubkey(x_pubkey): - return x_pubkey[0:2] in ['fe', 'ff'] +def hash_encode(x): + return x[::-1].encode('hex') diff --git a/lbrynet/wallet/wallet.py b/lbrynet/wallet/wallet.py index ea230007f..2cc7a8dee 100644 --- a/lbrynet/wallet/wallet.py +++ b/lbrynet/wallet/wallet.py @@ -1,72 +1,114 @@ -import copy import stat import json import os -import logging -from .constants import NEW_SEED_VERSION -from .account import Account -from .mnemonic import Mnemonic -from .lbrycrd import pw_encode, bip32_private_derivation, bip32_root -from .blockchain import BlockchainTransactions - -log = logging.getLogger(__name__) +from lbrynet.wallet.account import Account +from lbrynet.wallet.constants import MAIN_CHAIN -class WalletStorage: +class Wallet: - def __init__(self, path): - self.data = {} - self.path = path - self.file_exists = False - self.modified = False - self.path and self.read() + def __init__(self, **kwargs): + self.name = kwargs.get('name', 'Wallet') + self.chain = kwargs.get('chain', MAIN_CHAIN) + self.accounts = kwargs.get('accounts') or {0: Account.generate()} - def read(self): - try: - with open(self.path, "r") as f: - data = f.read() - except IOError: - return - try: - self.data = json.loads(data) - except Exception: - self.data = {} - raise IOError("Cannot read wallet file '%s'" % self.path) - self.file_exists = True + @classmethod + def from_json(cls, json_data): + if 'accounts' in json_data: + json_data = json_data.copy() + json_data['accounts'] = { + a_id: Account.from_json(a) for + a_id, a in json_data['accounts'].items() + } + return cls(**json_data) - def get(self, key, default=None): - v = self.data.get(key) - if v is None: - v = default + def to_json(self): + return { + 'name': self.name, + 'chain': self.chain, + 'accounts': { + a_id: a.to_json() for + a_id, a in self.accounts.items() + } + } + + @property + def default_account(self): + return self.accounts.get(0, None) + + @property + def addresses(self): + for account in self.accounts.values(): + for address in account.addresses: + yield address + + def ensure_enough_addresses(self): + return [ + address + for account in self.accounts.values() + for address in account.ensure_enough_addresses() + ] + + def get_private_key_for_address(self, address): + for account in self.accounts.values(): + private_key = account.get_private_key_for_address(address) + if private_key is not None: + return private_key + + +class EphemeralWalletStorage(dict): + + LATEST_VERSION = 2 + + def save(self): + return json.dumps(self, indent=4, sort_keys=True) + + def upgrade(self): + + def _rename_property(old, new): + if old in self: + old_value = self[old] + del self[old] + if new not in self: + self[new] = old_value + + if self.get('version', 1) == 1: # upgrade from version 1 to version 2 + # TODO: `addr_history` should actually be imported into SQLStorage and removed from wallet. + _rename_property('addr_history', 'history') + _rename_property('use_encryption', 'encrypted') + _rename_property('gap_limit', 'gap_limit_for_receiving') + self['version'] = 2 + + self.save() + + +class PermanentWalletStorage(EphemeralWalletStorage): + + def __init__(self, *args, **kwargs): + super(PermanentWalletStorage, self).__init__(*args, **kwargs) + self.path = None + + @classmethod + def from_path(cls, path): + if os.path.exists(path): + with open(path, "r") as f: + json_data = f.read() + json_dict = json.loads(json_data) + storage = cls(**json_dict) + if 'version' in storage and storage['version'] != storage.LATEST_VERSION: + storage.upgrade() else: - v = copy.deepcopy(v) - return v + storage = cls() + storage.path = path + return storage - def put(self, key, value): - try: - json.dumps(key) - json.dumps(value) - except: - return - if value is not None: - if self.data.get(key) != value: - self.modified = True - self.data[key] = copy.deepcopy(value) - elif key in self.data: - self.modified = True - self.data.pop(key) + def save(self): + json_data = super(PermanentWalletStorage, self).save() - def write(self): - self._write() - - def _write(self): - if not self.modified: - return - s = json.dumps(self.data, indent=4, sort_keys=True) temp_path = "%s.tmp.%s" % (self.path, os.getpid()) with open(temp_path, "w") as f: - f.write(s) + f.write(json_data) f.flush() os.fsync(f.fileno()) @@ -74,169 +116,12 @@ class WalletStorage: mode = os.stat(self.path).st_mode else: mode = stat.S_IREAD | stat.S_IWRITE - # perform atomic write on POSIX systems + try: os.rename(temp_path, self.path) except: os.remove(self.path) os.rename(temp_path, self.path) os.chmod(self.path, mode) - self.modified = False - def upgrade(self): - - def _rename_property(old, new): - if old in self.data: - old_value = self.data[old] - del self.data[old] - if new not in self.data: - self.data[new] = old_value - - _rename_property('addr_history', 'history') - _rename_property('use_encryption', 'encrypted') - - -class Wallet: - - root_name = 'x/' - root_derivation = 'm/' - gap_limit_for_change = 6 - - def __init__(self, path, headers): - self.storage = storage = WalletStorage(path) - storage.upgrade() - self.headers = headers - self.accounts = self._instantiate_accounts(storage.get('accounts', {})) - self.history = BlockchainTransactions(storage.get('history', {})) - self.master_public_keys = storage.get('master_public_keys', {}) - self.master_private_keys = storage.get('master_private_keys', {}) - self.gap_limit = storage.get('gap_limit', 20) - self.seed = storage.get('seed', '') - self.seed_version = storage.get('seed_version', NEW_SEED_VERSION) - self.encrypted = storage.get('encrypted', storage.get('use_encryption', False)) - self.claim_certificates = storage.get('claim_certificates', {}) - self.default_certificate_claim = storage.get('default_certificate_claim', None) - - def _instantiate_accounts(self, accounts): - instances = {} - for index, details in accounts.items(): - if 'xpub' in details: - instances[index] = Account( - details, self.gap_limit, self.gap_limit_for_change, self.is_address_old - ) - else: - log.error("cannot load account: {}".format(details)) - return instances - - @property - def exists(self): - return self.storage.file_exists - - @property - def default_account(self): - return self.accounts['0'] - - @property - def sequences(self): - for account in self.accounts.values(): - for sequence in account.sequences: - yield sequence - - @property - def addresses(self): - for sequence in self.sequences: - for address in sequence.addresses: - yield address - - @property - def receiving_addresses(self): - for account in self.accounts.values(): - for address in account.receiving.addresses: - yield address - - @property - def change_addresses(self): - for account in self.accounts.values(): - for address in account.receiving.addresses: - yield address - - @property - def addresses_without_history(self): - for address in self.addresses: - if not self.history.has_address(address): - yield address - - def ensure_enough_addresses(self): - return [ - address - for sequence in self.sequences - for address in sequence.ensure_enough_addresses() - ] - - def create(self): - mnemonic = Mnemonic(self.storage.get('lang', 'eng')) - seed = mnemonic.make_seed() - self.add_seed(seed, None) - self.add_xprv_from_seed(seed, self.root_name, None) - account = Account( - {'xpub': self.master_public_keys.get("x/")}, - self.gap_limit, - self.gap_limit_for_change, - self.is_address_old - ) - self.add_account('0', account) - - def add_seed(self, seed, password): - if self.seed: - raise Exception("a seed exists") - self.seed_version, self.seed = self.format_seed(seed) - if password: - self.seed = pw_encode(self.seed, password) - self.storage.put('seed', self.seed) - self.storage.put('seed_version', self.seed_version) - self.set_use_encryption(password is not None) - - @staticmethod - def format_seed(seed): - return NEW_SEED_VERSION, ' '.join(seed.split()) - - def add_xprv_from_seed(self, seed, name, password, passphrase=''): - xprv, xpub = bip32_root(Mnemonic.mnemonic_to_seed(seed, passphrase)) - xprv, xpub = bip32_private_derivation(xprv, "m/", self.root_derivation) - self.add_master_public_key(name, xpub) - self.add_master_private_key(name, xprv, password) - - def add_master_public_key(self, name, xpub): - if xpub in self.master_public_keys.values(): - raise BaseException('Duplicate master public key') - self.master_public_keys[name] = xpub - self.storage.put('master_public_keys', self.master_public_keys) - - def add_master_private_key(self, name, xpriv, password): - self.master_private_keys[name] = pw_encode(xpriv, password) - self.storage.put('master_private_keys', self.master_private_keys) - - def add_account(self, account_id, account): - self.accounts[account_id] = account - self.save_accounts() - - def set_use_encryption(self, use_encryption): - self.use_encryption = use_encryption - self.storage.put('use_encryption', use_encryption) - - def save_accounts(self): - d = {} - for k, v in self.accounts.items(): - d[k] = v.as_dict() - self.storage.put('accounts', d) - - def is_address_old(self, address, age_limit=2): - age = -1 - for tx in self.history.get_transactions(address, []): - if tx.height == 0: - tx_age = 0 - else: - tx_age = self.headers.height - tx.height + 1 - if tx_age > age: - age = tx_age - return age > age_limit + return json_data diff --git a/requirements.txt b/requirements.txt index af06396d4..67c592e0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ txJSON-RPC==0.5 wsgiref==0.1.2 zope.interface==4.3.3 treq==17.8.0 +typing