import ast import copy import stat import json import os import random import threading import time import hashlib import logging from decimal import Decimal from functools import partial from lbryschema.address import hash_160_bytes_to_address, public_key_to_address, is_address from .account import Account from .constants import TYPE_ADDRESS, TYPE_CLAIM, TYPE_SUPPORT, TYPE_UPDATE, TYPE_PUBKEY from .constants import EXPIRATION_BLOCKS, COINBASE_MATURITY, RECOMMENDED_FEE from .coinchooser import COIN_CHOOSERS from .transaction import Transaction from .mnemonic import Mnemonic from .util import rev_hex from .errors import NotEnoughFunds, InvalidPassword from .constants import NEW_SEED_VERSION from .lbrycrd import regenerate_key, is_compressed, pw_encode, pw_decode from .lbrycrd import bip32_private_key from .lbrycrd import encode_claim_id_hex, deserialize_xkey, claim_id_hash from .lbrycrd import bip32_private_derivation, bip32_root log = logging.getLogger(__name__) class WalletStorage: def __init__(self, path): self.lock = threading.RLock() self.data = {} self.path = path self.file_exists = False self.modified = False log.info("wallet path: %s", self.path) if self.path: self.read(self.path) def read(self, path): """Read the contents of the wallet file.""" try: with open(self.path, "r") as f: data = f.read() except IOError: return try: self.data = json.loads(data) except: try: d = ast.literal_eval(data) # parse raw data from reading wallet file labels = d.get('labels', {}) except Exception as e: raise IOError("Cannot read wallet file '%s'" % self.path) self.data = {} # In old versions of Electrum labels were latin1 encoded, this fixes breakage. for i, label in labels.items(): try: unicode(label) except UnicodeDecodeError: d['labels'][i] = unicode(label.decode('latin1')) for key, value in d.items(): try: json.dumps(key) json.dumps(value) except: log.error('Failed to convert label to json format: {}'.format(key)) continue self.data[key] = value self.file_exists = True def get(self, key, default=None): with self.lock: v = self.data.get(key) if v is None: v = default else: v = copy.deepcopy(v) return v def put(self, key, value): try: json.dumps(key) json.dumps(value) except: self.print_error("json error: cannot save", key) return with self.lock: if value is not None: if self.data.get(key) != value: self.modified = True self.data[key] = copy.deepcopy(value) elif key in self.data: self.modified = True self.data.pop(key) def write(self): with self.lock: self._write() def _write(self): if threading.currentThread().isDaemon(): log.warning('daemon thread cannot write wallet') return if not self.modified: return s = json.dumps(self.data, indent=4, sort_keys=True) temp_path = "%s.tmp.%s" % (self.path, os.getpid()) with open(temp_path, "w") as f: f.write(s) f.flush() os.fsync(f.fileno()) if os.path.exists(self.path): mode = os.stat(self.path).st_mode else: mode = stat.S_IREAD | stat.S_IWRITE # perform atomic write on POSIX systems try: os.rename(temp_path, self.path) except: os.remove(self.path) os.rename(temp_path, self.path) os.chmod(self.path, mode) self.modified = False class Wallet: root_name = 'x/' root_derivation = "m/" wallet_type = 'standard' max_change_outputs = 3 def __init__(self, path): self.storage = storage = WalletStorage(path) self.gap_limit = storage.get('gap_limit', 20) self.gap_limit_for_change = 6 self.accounts = {} self.seed_version = storage.get('seed_version', NEW_SEED_VERSION) self.use_change = storage.get('use_change', True) self.multiple_change = storage.get('multiple_change', False) self.use_encryption = storage.get('use_encryption', False) self.seed = storage.get('seed', '') # encrypted self.labels = storage.get('labels', {}) self.frozen_addresses = set(storage.get('frozen_addresses', [])) self.stored_height = storage.get('stored_height', 0) # last known height (for offline mode) self.history = storage.get('addr_history', {}) # address -> list(txid, height) # Transactions pending verification. A map from tx hash to transaction # height. Access is not contended so no lock is needed. self.unverified_tx = {} # Verified transactions. Each value is a (height, timestamp, block_pos) tuple. # Access with self.lock. self.verified_tx = storage.get('verified_tx3', {}) # there is a difference between wallet.up_to_date and interface.is_up_to_date() # interface.is_up_to_date() returns true when all requests have been answered and processed # wallet.up_to_date is true when the wallet is synchronized (stronger requirement) self.up_to_date = False self.claim_certificates = storage.get('claim_certificates', {}) self.default_certificate_claim = storage.get('default_certificate_claim', None) # save wallet type the first time if self.storage.get('wallet_type') is None: self.storage.put('wallet_type', self.wallet_type) self.master_public_keys = storage.get('master_public_keys', {}) self.master_private_keys = storage.get('master_private_keys', {}) self.mnemonic = Mnemonic(storage.get('lang', 'eng')) @property def addresses(self): for account in self.accounts.values(): for sequence in account.sequences: for address in sequence.addresses: yield address def create(self): seed = self.mnemonic.make_seed() self.add_seed(seed, None) self.add_xprv_from_seed(seed, self.root_name, None) self.add_account('0', Account({ 'xpub': self.master_public_keys.get("x/") }, self.gap_limit, self.gap_limit_for_change, self.address_is_old )) self.ensure_enough_addresses() def ensure_enough_addresses(self): for account in self.accounts.values(): account.ensure_enough_addresses() def load(self): self.load_accounts() self.load_transactions() def load_accounts(self): for index, details in self.storage.get('accounts', {}).items(): if 'xpub' in details: self.accounts[index] = Account( details, self.gap_limit, self.gap_limit_for_change, self.address_is_old ) else: log.error("cannot load account: {}".format(details)) def load_transactions(self): self.txi = self.storage.get('txi', {}) self.txo = self.storage.get('txo', {}) self.pruned_txo = self.storage.get('pruned_txo', {}) tx_list = self.storage.get('transactions', {}) self.claimtrie_transactions = self.storage.get('claimtrie_transactions', {}) self.transactions = {} for tx_hash, raw in tx_list.items(): tx = Transaction(raw) self.transactions[tx_hash] = tx if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None and \ (tx_hash not in self.pruned_txo.values()): log.info("removing unreferenced tx: %s", tx_hash) self.transactions.pop(tx_hash) # add to claimtrie transactions if its a claimtrie transaction tx.deserialize() for n, txout in enumerate(tx.outputs()): if txout[0] & (TYPE_CLAIM | TYPE_UPDATE | TYPE_SUPPORT): self.claimtrie_transactions[tx_hash + ':' + str(n)] = txout[0] def set_use_encryption(self, use_encryption): self.use_encryption = use_encryption self.storage.put('use_encryption', use_encryption) def save_transactions(self, write=False): tx = {} for k, v in self.transactions.items(): tx[k] = str(v) self.storage.put('transactions', tx) self.storage.put('txi', self.txi) self.storage.put('txo', self.txo) self.storage.put('pruned_txo', self.pruned_txo) self.storage.put('addr_history', self.history) self.storage.put('claimtrie_transactions', self.claimtrie_transactions) if write: self.storage.write() def save_certificate(self, claim_id, private_key, write=False): certificate_keys = self.storage.get('claim_certificates') or {} certificate_keys[claim_id] = private_key self.storage.put('claim_certificates', certificate_keys) if write: self.storage.write() def set_default_certificate(self, claim_id, overwrite_existing=True, write=False): if self.default_certificate_claim is not None and overwrite_existing or not \ self.default_certificate_claim: self.storage.put('default_certificate_claim', claim_id) if write: self.storage.write() self.default_certificate_claim = claim_id def get_certificate_signing_key(self, claim_id): certificates = self.storage.get('claim_certificates', {}) return certificates.get(claim_id, None) def get_certificate_claim_ids_for_signing(self): certificates = self.storage.get('claim_certificates', {}) return certificates.keys() def clear_history(self): with self.transaction_lock: self.txi = {} self.txo = {} self.pruned_txo = {} self.save_transactions() with self.lock: self.history = {} self.tx_addr_hist = {} def build_reverse_history(self): self.tx_addr_hist = {} for addr, hist in self.history.items(): for tx_hash, h in hist: s = self.tx_addr_hist.get(tx_hash, set()) s.add(addr) self.tx_addr_hist[tx_hash] = s def check_history(self): save = False for addr, hist in self.history.items(): if not self.is_mine(addr): self.history.pop(addr) save = True continue for tx_hash, tx_height in hist: if tx_hash in self.pruned_txo.values() or self.txi.get(tx_hash) or self.txo.get( tx_hash): continue tx = self.transactions.get(tx_hash) if tx is not None: self.add_transaction(tx_hash, tx) save = True if save: self.save_transactions() def set_up_to_date(self, up_to_date): with self.lock: self.up_to_date = up_to_date if up_to_date: self.save_transactions(write=True) def is_up_to_date(self): with self.lock: return self.up_to_date def set_label(self, name, text=None): changed = False old_text = self.labels.get(name) if text: if old_text != text: self.labels[name] = text changed = True else: if old_text: self.labels.pop(name) changed = True if changed: self.storage.put('labels', self.labels) return changed def is_mine(self, address): return address in self.addresses def is_change(self, address): if not self.is_mine(address): return False acct, s = self.get_address_index(address) if s is None: return False return s[0] == 1 def get_address_index(self, address): for acc_id in self.accounts: for for_change in [0, 1]: addresses = self.accounts[acc_id].get_addresses(for_change) if address in addresses: return acc_id, (for_change, addresses.index(address)) raise Exception("Address not found", address) def get_private_key(self, address, password): if self.is_watching_only(): return [] account_id, sequence = self.get_address_index(address) return self.accounts[account_id].get_private_key(sequence, self, password) def get_public_keys(self, address): account_id, sequence = self.get_address_index(address) return self.accounts[account_id].get_pubkeys(*sequence) def sign_message(self, address, message, password): keys = self.get_private_key(address, password) assert len(keys) == 1 sec = keys[0] key = regenerate_key(sec) compressed = is_compressed(sec) return key.sign_message(message, compressed, address) def decrypt_message(self, pubkey, message, password): address = public_key_to_address(pubkey.decode('hex')) keys = self.get_private_key(address, password) secret = keys[0] ec = regenerate_key(secret) decrypted = ec.decrypt_message(message) return decrypted def add_unverified_tx(self, tx_hash, tx_height): # Only add if confirmed and not verified if tx_height > 0 and tx_hash not in self.verified_tx: self.unverified_tx[tx_hash] = tx_height def add_verified_tx(self, tx_hash, info): # Remove from the unverified map and add to the verified map and self.unverified_tx.pop(tx_hash, None) with self.lock: self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos) self.storage.put('verified_tx3', self.verified_tx) conf, timestamp = self.get_confirmations(tx_hash) self.network.trigger_callback('verified', tx_hash, conf, timestamp) def get_unverified_txs(self): """Returns a map from tx hash to transaction height""" return self.unverified_tx def undo_verifications(self, height): """Used by the verifier when a reorg has happened""" txs = [] with self.lock: for tx_hash, item in self.verified_tx: tx_height, timestamp, pos = item if tx_height >= height: self.verified_tx.pop(tx_hash, None) txs.append(tx_hash) return txs def get_local_height(self): """ return last known height if we are offline """ return self.network.get_local_height() if self.network else self.stored_height def get_confirmations(self, tx): """ return the number of confirmations of a monitored transaction. """ with self.lock: if tx in self.verified_tx: height, timestamp, pos = self.verified_tx[tx] conf = (self.get_local_height() - height + 1) if conf <= 0: timestamp = None elif tx in self.unverified_tx: conf = -1 timestamp = None else: conf = 0 timestamp = None return conf, timestamp def get_txpos(self, tx_hash): "return position, even if the tx is unverified" with self.lock: x = self.verified_tx.get(tx_hash) y = self.unverified_tx.get(tx_hash) if x: height, timestamp, pos = x return height, pos elif y: return y, 0 else: return 1e12, 0 def is_found(self): return self.history.values() != [[]] * len(self.history) def get_num_tx(self, address): """ return number of transactions where address is involved """ return len(self.history.get(address, [])) def get_tx_delta(self, tx_hash, address): "effect of tx on address" # pruned if tx_hash in self.pruned_txo.values(): return None delta = 0 # substract the value of coins sent from address d = self.txi.get(tx_hash, {}).get(address, []) for n, v in d: delta -= v # add the value of the coins received at address d = self.txo.get(tx_hash, {}).get(address, []) for n, v, cb in d: delta += v return delta def get_wallet_delta(self, tx): """ effect of tx on wallet """ addresses = self.addresses is_relevant = False is_send = False is_pruned = False is_partial = False v_in = v_out = v_out_mine = 0 for item in tx.inputs(): addr = item.get('address') if addr in addresses: is_send = True is_relevant = True d = self.txo.get(item['prevout_hash'], {}).get(addr, []) for n, v, cb in d: if n == item['prevout_n']: value = v break else: value = None if value is None: is_pruned = True else: v_in += value else: is_partial = True if not is_send: is_partial = False for addr, value in tx.get_outputs(): v_out += value if addr in addresses: v_out_mine += value is_relevant = True if is_pruned: # some inputs are mine: fee = None if is_send: v = v_out_mine - v_out else: # no input is mine v = v_out_mine else: v = v_out_mine - v_in if is_partial: # some inputs are mine, but not all fee = None is_send = v < 0 else: # all inputs are mine fee = v_out - v_in return is_relevant, is_send, v, fee def get_addr_io(self, address): h = self.history.get(address, []) received = {} sent = {} for tx_hash, height in h: l = self.txo.get(tx_hash, {}).get(address, []) for n, v, is_cb in l: received[tx_hash + ':%d' % n] = (height, v, is_cb) for tx_hash, height in h: l = self.txi.get(tx_hash, {}).get(address, []) for txi, v in l: sent[txi] = height return received, sent def get_addr_utxo(self, address): coins, spent = self.get_addr_io(address) for txi in spent: coins.pop(txi) return coins # return the total amount ever received by an address def get_addr_received(self, address): received, sent = self.get_addr_io(address) return sum([v for height, v, is_cb in received.values()]) # return the balance of a bitcoin address: confirmed and matured, unconfirmed, unmatured def get_addr_balance(self, address, exclude_claimtrietx=False): received, sent = self.get_addr_io(address) c = u = x = 0 for txo, (tx_height, v, is_cb) in received.items(): exclude_tx = False # check if received transaction is a claimtrie tx to ourself if exclude_claimtrietx: prevout_hash, prevout_n = txo.split(':') tx_type = self.claimtrie_transactions.get(txo) if tx_type is not None: exclude_tx = True if not exclude_tx: if is_cb and tx_height + COINBASE_MATURITY > self.get_local_height(): x += v elif tx_height > 0: c += v else: u += v if txo in sent: if sent[txo] > 0: c -= v else: u -= v return c, u, x # get coin object in order to abandon calimtrie transactions # equivalent of get_spendable_coins but for claimtrie utxos def get_spendable_claimtrietx_coin(self, txid, nOut): tx = self.transactions.get(txid) if tx is None: raise BaseException('txid was not found in wallet') tx.deserialize() txouts = tx.outputs() if len(txouts) < nOut + 1: raise BaseException('nOut is too large') txout = txouts[nOut] txout_type, txout_dest, txout_value = txout if not txout_type & (TYPE_CLAIM | TYPE_UPDATE | TYPE_SUPPORT): raise BaseException('txid and nOut does not refer to a claimtrie transaction') address = txout_dest[1] utxos = self.get_addr_utxo(address) try: utxo = utxos[txid + ':' + str(nOut)] except KeyError: raise BaseException('this claimtrie transaction has already been spent') # create inputs is_update = txout_type & TYPE_UPDATE is_claim = txout_type & TYPE_CLAIM is_support = txout_type & TYPE_SUPPORT i = {'prevout_hash': txid, 'prevout_n': nOut, 'address': address, 'value': txout_value, 'is_update': is_update, 'is_claim': is_claim, 'is_support': is_support, 'height': utxo[0]} if is_claim: i['claim_name'] = txout_dest[0][0] i['claim_value'] = txout_dest[0][1] elif is_support: i['claim_name'] = txout_dest[0][0] i['claim_id'] = txout_dest[0][1] elif is_update: i['claim_name'] = txout_dest[0][0] i['claim_id'] = txout_dest[0][1] i['claim_value'] = txout_dest[0][2] else: # should not reach here raise ZeroDivisionError() self.add_input_info(i) return i def get_spendable_coins(self, domain=None, exclude_frozen=True, abandon_txid=None): coins = [] found_abandon_txid = False if domain is None: domain = list(self.addresses) if exclude_frozen: domain = set(domain) - self.frozen_addresses for addr in domain: c = self.get_addr_utxo(addr) for txo, v in c.items(): tx_height, value, is_cb = v if is_cb and tx_height + COINBASE_MATURITY > self.get_local_height(): continue prevout_hash, prevout_n = txo.split(':') tx = self.transactions.get(prevout_hash) tx.deserialize() txout = tx.outputs()[int(prevout_n)] if txout[0] & (TYPE_CLAIM | TYPE_SUPPORT | TYPE_UPDATE) == 0 or ( abandon_txid is not None and prevout_hash == abandon_txid): output = { 'address': addr, 'value': value, 'prevout_n': int(prevout_n), 'prevout_hash': prevout_hash, 'height': tx_height, 'coinbase': is_cb, 'is_claim': bool(txout[0] & TYPE_CLAIM), 'is_support': bool(txout[0] & TYPE_SUPPORT), 'is_update': bool(txout[0] & TYPE_UPDATE), } if txout[0] & TYPE_CLAIM: output['claim_name'] = txout[1][0][0] output['claim_value'] = txout[1][0][1] elif txout[0] & TYPE_SUPPORT: output['claim_name'] = txout[1][0][0] output['claim_id'] = txout[1][0][1] elif txout[0] & TYPE_UPDATE: output['claim_name'] = txout[1][0][0] output['claim_id'] = txout[1][0][1] output['claim_value'] = txout[1][0][2] coins.append(output) if abandon_txid is not None and prevout_hash == abandon_txid: found_abandon_txid = True continue if abandon_txid is not None and not found_abandon_txid: raise ValueError("Can't spend from the given txid") return coins def get_account_addresses(self, acc_id, include_change=True): '''acc_id of None means all user-visible accounts''' addr_list = [] acc_ids = self.accounts_to_show() if acc_id is None else [acc_id] for _acc_id in acc_ids: if _acc_id in self.accounts: acc = self.accounts[_acc_id] addr_list += acc.get_addresses(0) if include_change: addr_list += acc.get_addresses(1) return addr_list def get_account_from_address(self, addr): """Returns the account that contains this address, or None""" for acc_id in self.accounts: # similar to get_address_index but simpler if addr in self.get_account_addresses(acc_id): return acc_id return None def get_account_balance(self, account, exclude_claimtrietx=False): return self.get_balance(self.get_account_addresses(account, exclude_claimtrietx)) def get_frozen_balance(self): return self.get_balance(self.frozen_addresses) def get_balance(self, domain=None, exclude_claimtrietx=False): if domain is None: domain = self.addresses(True) cc = uu = xx = 0 for addr in domain: c, u, x = self.get_addr_balance(addr, exclude_claimtrietx) cc += c uu += u xx += x return cc, uu, xx def get_address_history(self, address): with self.lock: return self.history.get(address, []) def get_status(self, h): if not h: return None status = '' for tx_hash, height in h: status += tx_hash + ':%d:' % height return hashlib.sha256(status).digest().encode('hex') def find_pay_to_pubkey_address(self, prevout_hash, prevout_n): dd = self.txo.get(prevout_hash, {}) for addr, l in dd.items(): for n, v, is_cb in l: if n == prevout_n: self.print_error("found pay-to-pubkey address:", addr) return addr def add_transaction(self, tx_hash, tx): log.info("Adding tx: %s", tx_hash) is_coinbase = True if tx.inputs()[0].get('is_coinbase') else False with self.transaction_lock: # add inputs self.txi[tx_hash] = d = {} for txi in tx.inputs(): addr = txi.get('address') if not txi.get('is_coinbase'): prevout_hash = txi['prevout_hash'] prevout_n = txi['prevout_n'] ser = prevout_hash + ':%d' % prevout_n if addr == "(pubkey)": addr = self.find_pay_to_pubkey_address(prevout_hash, prevout_n) # find value from prev output if addr and self.is_mine(addr): dd = self.txo.get(prevout_hash, {}) for n, v, is_cb in dd.get(addr, []): if n == prevout_n: if d.get(addr) is None: d[addr] = [] d[addr].append((ser, v)) break else: self.pruned_txo[ser] = tx_hash # add outputs self.txo[tx_hash] = d = {} for n, txo in enumerate(tx.outputs()): ser = tx_hash + ':%d' % n _type, x, v = txo if _type & (TYPE_CLAIM | TYPE_UPDATE | TYPE_SUPPORT): x = x[1] self.claimtrie_transactions[ser] = _type if _type & TYPE_ADDRESS: addr = x elif _type & TYPE_PUBKEY: addr = public_key_to_address(x.decode('hex')) else: addr = None if addr and self.is_mine(addr): if d.get(addr) is None: d[addr] = [] d[addr].append((n, v, is_coinbase)) # give v to txi that spends me next_tx = self.pruned_txo.get(ser) if next_tx is not None: self.pruned_txo.pop(ser) dd = self.txi.get(next_tx, {}) if dd.get(addr) is None: dd[addr] = [] dd[addr].append((ser, v)) # save self.transactions[tx_hash] = tx log.info("Saved") def remove_transaction(self, tx_hash): with self.transaction_lock: self.print_error("removing tx from history", tx_hash) # tx = self.transactions.pop(tx_hash) for ser, hh in self.pruned_txo.items(): if hh == tx_hash: self.pruned_txo.pop(ser) # add tx to pruned_txo, and undo the txi addition for next_tx, dd in self.txi.items(): for addr, l in dd.items(): ll = l[:] for item in ll: ser, v = item prev_hash, prev_n = ser.split(':') if prev_hash == tx_hash: l.remove(item) self.pruned_txo[ser] = next_tx if not l: dd.pop(addr) else: dd[addr] = l try: self.txi.pop(tx_hash) self.txo.pop(tx_hash) except KeyError: self.print_error("tx was not in history", tx_hash) def receive_tx_callback(self, tx_hash, tx, tx_height): self.add_transaction(tx_hash, tx) self.save_transactions() self.add_unverified_tx(tx_hash, tx_height) def receive_history_callback(self, addr, hist): with self.lock: old_hist = self.history.get(addr, []) for tx_hash, height in old_hist: if (tx_hash, height) not in hist: # remove tx if it's not referenced in histories self.tx_addr_hist[tx_hash].remove(addr) if not self.tx_addr_hist[tx_hash]: self.remove_transaction(tx_hash) self.history[addr] = hist for tx_hash, tx_height in hist: # add it in case it was previously unconfirmed self.add_unverified_tx(tx_hash, tx_height) # add reference in tx_addr_hist s = self.tx_addr_hist.get(tx_hash, set()) s.add(addr) self.tx_addr_hist[tx_hash] = s # if addr is new, we have to recompute txi and txo tx = self.transactions.get(tx_hash) if tx is not None and self.txi.get(tx_hash, {}).get(addr) is None and self.txo.get( tx_hash, {}).get(addr) is None: self.add_transaction(tx_hash, tx) # Write updated TXI, TXO etc. self.save_transactions() def get_history(self, domain=None): from collections import defaultdict # get domain if domain is None: domain = self.get_account_addresses(None) # 1. Get the history of each address in the domain, maintain the # delta of a tx as the sum of its deltas on domain addresses tx_deltas = defaultdict(int) for addr in domain: h = self.get_address_history(addr) for tx_hash, height in h: delta = self.get_tx_delta(tx_hash, addr) if delta is None or tx_deltas[tx_hash] is None: tx_deltas[tx_hash] = None else: tx_deltas[tx_hash] += delta # 2. create sorted history history = [] for tx_hash, delta in tx_deltas.items(): conf, timestamp = self.get_confirmations(tx_hash) history.append((tx_hash, conf, delta, timestamp)) history.sort(key=lambda x: self.get_txpos(x[0])) history.reverse() # 3. add balance c, u, x = self.get_balance(domain) balance = c + u + x h2 = [] for item in history: tx_hash, conf, delta, timestamp = item h2.append((tx_hash, conf, delta, timestamp, balance)) if balance is None or delta is None: balance = None else: balance -= delta h2.reverse() # fixme: this may happen if history is incomplete if balance not in [None, 0]: self.print_error("Error: history not synchronized") return [] return h2 def get_name_claims(self, domain=None, include_abandoned=True, include_supports=True, exclude_expired=True): claims = [] if domain is None: domain = self.get_account_addresses(None) for addr in domain: txos, txis = self.get_addr_io(addr) for txo, v in txos.items(): tx_height, value, is_cb = v prevout_hash, prevout_n = txo.split(':') tx = self.transactions.get(prevout_hash) tx.deserialize() txout = tx.outputs()[int(prevout_n)] if not include_abandoned and txo in txis: continue if not include_supports and txout[0] & TYPE_SUPPORT: continue if txout[0] & (TYPE_CLAIM | TYPE_UPDATE | TYPE_SUPPORT): local_height = self.get_local_height() expired = tx_height + EXPIRATION_BLOCKS <= local_height if expired and exclude_expired: continue output = { 'txid': prevout_hash, 'nout': int(prevout_n), 'address': addr, 'amount': Decimal(value), 'height': tx_height, 'expiration_height': tx_height + EXPIRATION_BLOCKS, 'expired': expired, 'confirmations': local_height - tx_height, 'is_spent': txo in txis, } if tx_height: output['height'] = tx_height output['expiration_height'] = tx_height + EXPIRATION_BLOCKS output['expired'] = expired output['confirmations'] = local_height - tx_height output['is_pending'] = False else: output['height'] = None output['expiration_height'] = None output['expired'] = expired output['confirmations'] = None output['is_pending'] = True if txout[0] & TYPE_CLAIM: output['category'] = 'claim' claim_name, claim_value = txout[1][0] output['name'] = claim_name output['value'] = claim_value.encode('hex') claim_id = claim_id_hash(rev_hex(output['txid']).decode('hex'), output['nout']) claim_id = encode_claim_id_hex(claim_id) output['claim_id'] = claim_id elif txout[0] & TYPE_SUPPORT: output['category'] = 'support' claim_name, claim_id = txout[1][0] output['name'] = claim_name output['claim_id'] = encode_claim_id_hex(claim_id) elif txout[0] & TYPE_UPDATE: output['category'] = 'update' claim_name, claim_id, claim_value = txout[1][0] output['name'] = claim_name output['value'] = claim_value.encode('hex') output['claim_id'] = encode_claim_id_hex(claim_id) if not expired: output[ 'blocks_to_expiration'] = tx_height + EXPIRATION_BLOCKS - local_height claims.append(output) return claims def get_label(self, tx_hash): label = self.labels.get(tx_hash, '') if label == '': label = self.get_default_label(tx_hash) return label def get_default_label(self, tx_hash): if self.txi.get(tx_hash) == {}: d = self.txo.get(tx_hash, {}) labels = [] for addr in d.keys(): label = self.labels.get(addr) if label: labels.append(label) return ', '.join(labels) return '' def fee_per_kb(self, config): b = config.get('dynamic_fees') f = config.get('fee_factor', 50) F = config.get('fee_per_kb', RECOMMENDED_FEE) if b and self.network and self.network.fee: result = min(RECOMMENDED_FEE, self.network.fee * (50 + f) / 100) else: result = F return result def relayfee(self): RELAY_FEE = 5000 MAX_RELAY_FEE = 50000 f = self.network.relay_fee if self.network and self.network.relay_fee else RELAY_FEE return min(f, MAX_RELAY_FEE) def get_tx_fee(self, tx): # this method can be overloaded return tx.get_fee() def coin_chooser_name(self, config): kind = config.get('coin_chooser') if kind not in COIN_CHOOSERS: kind = 'Priority' return kind def coin_chooser(self, config): klass = COIN_CHOOSERS[self.coin_chooser_name(config)] return klass() def make_unsigned_transaction(self, coins, outputs, config, fixed_fee=None, change_addr=None, abandon_txid=None): # check outputs for type, data, value in outputs: if type & (TYPE_CLAIM | TYPE_UPDATE | TYPE_SUPPORT): data = data[1] if type & TYPE_ADDRESS: assert is_address(data), "Address " + data + " is invalid!" # Avoid index-out-of-range with coins[0] below if not coins: raise NotEnoughFunds() for item in coins: self.add_input_info(item) # change address if change_addr: change_addrs = [change_addr] else: # send change to one of the accounts involved in the tx address = coins[0].get('address') account, _ = self.get_address_index(address) if self.use_change and self.accounts[account].has_change(): # New change addresses are created only after a few # confirmations. Select the unused addresses within the # gap limit; if none take one at random addrs = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change:] change_addrs = [addr for addr in addrs if self.get_num_tx(addr) == 0] if not change_addrs: change_addrs = [random.choice(addrs)] else: change_addrs = [address] # Fee estimator if fixed_fee is None: fee_estimator = partial(Transaction.fee_for_size, self.relayfee(), self.fee_per_kb(config)) else: fee_estimator = lambda size: fixed_fee # Change <= dust threshold is added to the tx fee dust_threshold = 182 * 3 * self.relayfee() / 1000 # Let the coin chooser select the coins to spend max_change = self.max_change_outputs if self.multiple_change else 1 coin_chooser = self.coin_chooser(config) tx = coin_chooser.make_tx(coins, outputs, change_addrs[:max_change], fee_estimator, dust_threshold, abandon_txid=abandon_txid) # Sort the inputs and outputs deterministically tx.BIP_LI01_sort() return tx def mktx(self, outputs, password, config, fee=None, change_addr=None, domain=None): coins = self.get_spendable_coins(domain) tx = self.make_unsigned_transaction(coins, outputs, config, fee, change_addr) self.sign_transaction(tx, password) return tx def add_input_info(self, txin): address = txin['address'] account_id, sequence = self.get_address_index(address) account = self.accounts[account_id] redeemScript = account.redeem_script(*sequence) pubkeys = account.get_pubkeys(*sequence) x_pubkeys = account.get_xpubkeys(*sequence) # sort pubkeys and x_pubkeys, using the order of pubkeys pubkeys, x_pubkeys = zip(*sorted(zip(pubkeys, x_pubkeys))) txin['pubkeys'] = list(pubkeys) txin['x_pubkeys'] = list(x_pubkeys) txin['signatures'] = [None] * len(pubkeys) if redeemScript: txin['redeemScript'] = redeemScript txin['num_sig'] = account.m else: txin['redeemPubkey'] = account.get_pubkey(*sequence) txin['num_sig'] = 1 def sign_transaction(self, tx, password): if self.is_watching_only(): return # Raise if password is not correct. self.check_password(password) # Add derivation for utxo in wallets for i, addr in self.utxo_can_sign(tx): txin = tx.inputs()[i] txin['address'] = addr self.add_input_info(txin) # Add private keys keypairs = {} for x in self.xkeys_can_sign(tx): sec = self.get_private_key_from_xpubkey(x, password) if sec: keypairs[x] = sec # Sign if keypairs: tx.sign(keypairs) def send_tx(self, tx, timeout=300): # fixme: this does not handle the case where server does not answer if not self.network.interface: raise Exception("Not connected.") txid = tx.hash() with self.send_tx_lock: self.network.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast) self.tx_event.wait() success, result = self.receive_tx(txid, tx) self.tx_event.clear() if not success: log.error("send tx failed: %s", result) return success, result log.debug("waiting for %s to be added to the wallet", txid) now = time.time() while txid not in self.transactions and time.time() < now + timeout: time.sleep(0.2) if txid not in self.transactions: #TODO: detect if the txid is not known because it changed log.error("timed out while waiting to receive back a broadcast transaction, " "expected txid: %s", txid) return False, "timed out while waiting to receive back a broadcast transaction, " \ "expected txid: %s" % txid log.info("successfully sent %s", txid) return success, result def on_broadcast(self, r): self.tx_result = r.get('result') self.tx_event.set() def receive_tx(self, tx_hash, tx): out = self.tx_result if out != tx_hash: return False, "error: " + out return True, out def update_password(self, old_password, new_password): if new_password == '': new_password = None if self.has_seed(): decoded = self.get_seed(old_password) self.seed = pw_encode(decoded, new_password) self.storage.put('seed', self.seed) if hasattr(self, 'master_private_keys'): for k, v in self.master_private_keys.items(): b = pw_decode(v, old_password) c = pw_encode(b, new_password) self.master_private_keys[k] = c self.storage.put('master_private_keys', self.master_private_keys) self.set_use_encryption(new_password is not None) def is_frozen(self, addr): return addr in self.frozen_addresses def set_frozen_state(self, addrs, freeze): '''Set frozen state of the addresses to FREEZE, True or False''' if all(self.is_mine(addr) for addr in addrs): if freeze: self.frozen_addresses |= set(addrs) else: self.frozen_addresses -= set(addrs) self.storage.put('frozen_addresses', list(self.frozen_addresses)) return True return False def prepare_for_verifier(self): # review transactions that are in the history for addr, hist in self.history.items(): for tx_hash, tx_height in hist: # add it in case it was previously unconfirmed self.add_unverified_tx(tx_hash, tx_height) # if we are on a pruning server, remove unverified transactions vr = self.verified_tx.keys() + self.unverified_tx.keys() for tx_hash in self.transactions.keys(): if tx_hash not in vr: log.info("removing transaction %s", tx_hash) self.transactions.pop(tx_hash) def accounts_to_show(self): return self.accounts.keys() def get_accounts(self): return {a_id: a for a_id, a in self.accounts.items() if a_id in self.accounts_to_show()} def get_account_name(self, k): default_name = "Main account" if k == '0' else "Account " + k return self.labels.get(k, default_name) def get_account_names(self): ids = self.accounts_to_show() return dict(zip(ids, map(self.get_account_name, ids))) def add_account(self, account_id, account): self.accounts[account_id] = account self.save_accounts() def save_accounts(self): d = {} for k, v in self.accounts.items(): d[k] = v.dump() self.storage.put('accounts', d) def is_used(self, address): h = self.history.get(address, []) c, u, x = self.get_addr_balance(address) return len(h) > 0 and c + u + x == 0 def is_empty(self, address): c, u, x = self.get_addr_balance(address) return c + u + x == 0 def address_is_old(self, address, age_limit=2): age = -1 h = self.history.get(address, []) for tx_hash, tx_height in h: if tx_height == 0: tx_age = 0 else: tx_age = self.get_local_height() - tx_height + 1 if tx_age > age: age = tx_age return age > age_limit def can_sign(self, tx): if self.is_watching_only(): return False if tx.is_complete(): return False if self.xkeys_can_sign(tx): return True if self.utxo_can_sign(tx): return True return False def utxo_can_sign(self, tx): out = set() coins = self.get_spendable_coins() for i in tx.inputs_without_script(): txin = tx.inputs[i] for item in coins: if txin.get('prevout_hash') == item.get('prevout_hash') and txin.get( 'prevout_n') == item.get('prevout_n'): out.add((i, item.get('address'))) return out def xkeys_can_sign(self, tx): out = set() for x in tx.inputs_to_sign(): if self.can_sign_xpubkey(x): out.add(x) return out def get_private_key_from_xpubkey(self, x_pubkey, password): if x_pubkey[0:2] in ['02', '03', '04']: addr = public_key_to_address(x_pubkey.decode('hex')) if self.is_mine(addr): return self.get_private_key(addr, password)[0] elif x_pubkey[0:2] == 'ff': xpub, sequence = Account.parse_xpubkey(x_pubkey) for k, v in self.master_public_keys.items(): if v == xpub: xprv = self.get_master_private_key(k, password) if xprv: _, _, _, c, k = deserialize_xkey(xprv) return bip32_private_key(sequence, k, c) elif x_pubkey[0:2] == 'fd': addrtype = ord(x_pubkey[2:4].decode('hex')) addr = hash_160_bytes_to_address(x_pubkey[4:].decode('hex'), addrtype) if self.is_mine(addr): return self.get_private_key(addr, password)[0] else: raise BaseException("z") def can_sign_xpubkey(self, x_pubkey): if x_pubkey[0:2] in ['02', '03', '04']: addr = public_key_to_address(x_pubkey.decode('hex')) return self.is_mine(addr) elif x_pubkey[0:2] == 'ff': if not isinstance(self, Wallet): return False xpub, sequence = Account.parse_xpubkey(x_pubkey) return xpub in [self.master_public_keys[k] for k in self.master_private_keys.keys()] elif x_pubkey[0:2] == 'fd': addrtype = ord(x_pubkey[2:4].decode('hex')) addr = hash_160_bytes_to_address(x_pubkey[4:].decode('hex'), addrtype) return self.is_mine(addr) else: raise BaseException("z") def can_change_password(self): return not self.is_watching_only() def get_unused_addresses(self, account): # fixme: use slots from expired requests domain = self.get_account_addresses(account, include_change=False) return [addr for addr in domain if not self.history.get(addr)] def get_unused_address(self, account): domain = self.get_account_addresses(account, include_change=False) for addr in domain: if not self.history.get(addr): return addr def is_watching_only(self): return not bool(self.master_private_keys) def get_master_public_key(self): return self.master_public_keys.get(self.root_name) def get_master_private_key(self, account, password): k = self.master_private_keys.get(account) if not k: return xprv = pw_decode(k, password) try: deserialize_xkey(xprv) except: raise InvalidPassword() return xprv def check_password(self, password): xpriv = self.get_master_private_key(self.root_name, password) xpub = self.master_public_keys[self.root_name] if deserialize_xkey(xpriv)[3] != deserialize_xkey(xpub)[3]: raise InvalidPassword() def add_master_public_key(self, name, xpub): if xpub in self.master_public_keys.values(): raise BaseException('Duplicate master public key') self.master_public_keys[name] = xpub self.storage.put('master_public_keys', self.master_public_keys) def add_master_private_key(self, name, xpriv, password): self.master_private_keys[name] = pw_encode(xpriv, password) self.storage.put('master_private_keys', self.master_private_keys) def derive_xkeys(self, root, derivation, password): x = self.master_private_keys[root] root_xprv = pw_decode(x, password) xprv, xpub = bip32_private_derivation(root_xprv, root, derivation) return xpub, xprv def mnemonic_to_seed(self, seed, password): return Mnemonic.mnemonic_to_seed(seed, password) def format_seed(self, seed): return NEW_SEED_VERSION, ' '.join(seed.split()) @classmethod def account_derivation(cls, account_id): return cls.root_derivation + account_id @classmethod def address_derivation(cls, account_id, change, address_index): account_derivation = cls.account_derivation(account_id) return "%s/%d/%d" % (account_derivation, change, address_index) def address_id(self, address): acc_id, (change, address_index) = self.get_address_index(address) return self.address_derivation(acc_id, change, address_index) def add_xprv_from_seed(self, seed, name, password, passphrase=''): # we don't store the seed, only the master xpriv xprv, _ = bip32_root(self.mnemonic_to_seed(seed, passphrase)) xprv, xpub = bip32_private_derivation(xprv, "m/", self.root_derivation) self.add_master_public_key(name, xpub) self.add_master_private_key(name, xprv, password) def add_xpub_from_seed(self, seed, name): # store only master xpub xprv, _ = bip32_root(self.mnemonic_to_seed(seed, '')) _, xpub = bip32_private_derivation(xprv, "m/", self.root_derivation) self.add_master_public_key(name, xpub) def has_seed(self): return self.seed != '' def add_seed(self, seed, password): if self.seed: raise Exception("a seed exists") self.seed_version, self.seed = self.format_seed(seed) if password: self.seed = pw_encode(self.seed, password) self.storage.put('seed', self.seed) self.storage.put('seed_version', self.seed_version) self.set_use_encryption(password is not None) def get_seed(self, password): return pw_decode(self.seed, password) def get_mnemonic(self, password): return self.get_seed(password) def num_unused_trailing_addresses(self, addresses): k = 0 for a in addresses[::-1]: if self.history.get(a): break k = k + 1 return k def min_acceptable_gap(self): # fixme: this assumes wallet is synchronized n = 0 nmax = 0 for account in self.accounts.values(): addresses = account.get_addresses(0) k = self.num_unused_trailing_addresses(addresses) for a in addresses[0:-k]: if self.history.get(a): n = 0 else: n += 1 if n > nmax: nmax = n return nmax + 1 def default_account(self): return self.accounts['0'] def create_new_address(self, account=None, for_change=0): with self.lock: if account is None: account = self.default_account() address = account.create_new_address(for_change) self.add_address(address) log.info("created address %s", address) return address def add_address(self, address): if address not in self.history: self.history[address] = [] if self.synchronizer: self.synchronizer.add(address) self.save_accounts() def get_least_used_address(self, account=None, for_change=False, max_count=100): domain = self.get_account_addresses(account, include_change=for_change) hist = {} for addr in domain: if for_change != self.is_change(addr): continue else: h = self.history.get(addr) if h and len(h) >= max_count: continue elif h: hist[addr] = h else: hist[addr] = [] if hist: return sorted(hist.keys(), key=lambda x: len(hist[x]))[0] return self.create_new_address(account, for_change=for_change) def is_beyond_limit(self, address, account, is_change): addr_list = account.get_addresses(is_change) i = addr_list.index(address) prev_addresses = addr_list[:max(0, i)] limit = self.gap_limit_for_change if is_change else self.gap_limit if len(prev_addresses) < limit: return False prev_addresses = prev_addresses[max(0, i - limit):] for addr in prev_addresses: if self.history.get(addr): return False return True def get_master_public_keys(self): out = {} for k, account in self.accounts.items(): name = self.get_account_name(k) mpk_text = '\n\n'.join(account.get_master_pubkeys()) out[name] = mpk_text return out