From b1c246c01bfb18cdf0387a32cf9a03e02e849525 Mon Sep 17 00:00:00 2001 From: Josh Rickmar Date: Tue, 22 Oct 2013 09:55:53 -0400 Subject: [PATCH] Perform smarter UTXO tracking. This change fixes many issues with the tracking of unspent transaction outputs. First, notifications for when UTXOs arse spent are now requested from btcd, and when spent, will be removed from the UtxoStore. Second, when transactions are created, the unconfirmed (not yet in a block) Utxo (with block height -1 and zeroed block hash) is added to the wallet's UtxoStore. Notifications for when this UTXO is spent are also requested from btcd. After the tx appears in a block, because the UTXO has a pkScript to be spent by another owned wallet address, a notification with the UTXO will be sent to btcwallet. We already store the unconfirmed UTXO, so at this point the actual block height and hash are filled in. Finally, when calculating the balance, if confirmations is zero, unconfirmed UTXOs (block height -1) will be included in the balance. Otherwise, they are ignored. --- cmd.go | 108 +++++++++++++++++++++++++++++++++++++++++++++++++--- cmdmgr.go | 20 +++++++++- createtx.go | 54 ++++++++++++++++++-------- tx/tx.go | 4 ++ 4 files changed, 163 insertions(+), 23 deletions(-) diff --git a/cmd.go b/cmd.go index 97459f2..1682aa6 100644 --- a/cmd.go +++ b/cmd.go @@ -17,6 +17,7 @@ package main import ( + "bytes" "encoding/json" "errors" "fmt" @@ -59,11 +60,12 @@ var ( // to prevent against incorrect multiple access. type BtcWallet struct { *wallet.Wallet - mtx sync.RWMutex - name string - dirty bool - NewBlockTxSeqN uint64 - UtxoStore struct { + mtx sync.RWMutex + name string + dirty bool + NewBlockTxSeqN uint64 + SpentOutpointSeqN uint64 + UtxoStore struct { sync.RWMutex dirty bool s tx.UtxoStore @@ -276,7 +278,11 @@ func (w *BtcWallet) CalculateBalance(confirmations int) float64 { w.UtxoStore.RLock() for _, u := range w.UtxoStore.s { if int(height-u.Height) >= confirmations { - bal += u.Amt + // Utxos not yet in blocks (height -1) should only be + // added if confirmations is 0. + if u.Height != -1 || (confirmations == 0 && u.Height == -1) { + bal += u.Amt + } } } w.UtxoStore.RUnlock() @@ -298,6 +304,20 @@ func (w *BtcWallet) Track() { for _, addr := range w.GetActiveAddresses() { w.ReqNewTxsForAddress(addr) } + + n = <-NewJSONID + w.mtx.Lock() + w.SpentOutpointSeqN = n + w.mtx.Unlock() + + replyHandlers.Lock() + replyHandlers.m[n] = w.spentUtxoHandler + replyHandlers.Unlock() + w.UtxoStore.RLock() + for _, utxo := range w.UtxoStore.s { + w.ReqSpentUtxoNtfn(utxo) + } + w.UtxoStore.RUnlock() } // RescanForAddress requests btcd to rescan the blockchain for new @@ -358,6 +378,63 @@ func (w *BtcWallet) ReqNewTxsForAddress(addr string) { btcdMsgs <- msg } +// ReqSpentUtxoNtfn sends a message to btcd to request updates for when +// a stored UTXO has been spent. +func (w *BtcWallet) ReqSpentUtxoNtfn(u *tx.Utxo) { + log.Debugf("Requesting spent UTXO notifications for Outpoint hash %s index %d", + u.Out.Hash, u.Out.Index) + + w.mtx.RLock() + n := w.SpentOutpointSeqN + w.mtx.RUnlock() + + m := &btcjson.Message{ + Jsonrpc: "1.0", + Id: fmt.Sprintf("btcwallet(%d)", n), + Method: "notifyspent", + Params: []interface{}{ + u.Out.Hash.String(), + u.Out.Index, + }, + } + msg, _ := json.Marshal(m) + + btcdMsgs <- msg +} + +// spentUtxoHandler is the handler function for btcd spent UTXO notifications +// resulting from transactions in newly-attached blocks. +func (w *BtcWallet) spentUtxoHandler(result interface{}, e *btcjson.Error) bool { + if e != nil { + log.Errorf("Spent UTXO Handler: Error %d received from btcd: %s", + e.Code, e.Message) + return false + } + v, ok := result.(map[string]interface{}) + if !ok { + return false + } + txHashBE, ok := v["txhash"].(string) + if !ok { + log.Error("Spent UTXO Handler: Unspecified transaction hash.") + return false + } + txHash, err := btcwire.NewShaHashFromStr(txHashBE) + if err != nil { + log.Errorf("Spent UTXO Handler: Bad transaction hash: %s", err) + return false + } + index, ok := v["index"].(float64) + if !ok { + log.Error("Spent UTXO Handler: Unspecified index.") + } + + _, _ = txHash, index + + // Never remove this handler. + return false +} + // newBlockTxHandler is the handler function for btcd transaction // notifications resulting from newly-attached blocks. func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool { @@ -467,6 +544,25 @@ func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool // Do not add output to utxo store if spent. if !spent { go func() { + // First, iterate through all stored utxos. If an unconfirmed utxo + // (not present in a block) has the same outpoint as this utxo, + // update the block height and hash. + w.UtxoStore.RLock() + for _, u := range w.UtxoStore.s { + if u.Height != -1 { + continue + } + if bytes.Equal(u.Out.Hash[:], txhash[:]) && u.Out.Index == uint32(index) { + // Found it. + fmt.Println("omg everything worked.") + copy(u.BlockHash[:], blockhash[:]) + u.Height = int64(height) + w.UtxoStore.RUnlock() + return + } + } + w.UtxoStore.RUnlock() + u := &tx.Utxo{ Amt: uint64(amt), Height: int64(height), diff --git a/cmdmgr.go b/cmdmgr.go index 6c5df44..c01b380 100644 --- a/cmdmgr.go +++ b/cmdmgr.go @@ -460,7 +460,7 @@ func SendFrom(reply chan []byte, msg *btcjson.Message) { pairs := map[string]uint64{ toaddr58: uint64(amt), } - rawtx, inputs, err := w.txToPairs(pairs, uint64(fee), int(minconf)) + rawtx, inputs, changeUtxo, err := w.txToPairs(pairs, uint64(fee), int(minconf)) if err != nil { e := InternalError e.Message = err.Error() @@ -495,6 +495,14 @@ func SendFrom(reply chan []byte, msg *btcjson.Message) { // Remove previous unspent outputs now spent by the tx. w.UtxoStore.Lock() modified := w.UtxoStore.s.Remove(inputs) + + // Add unconfirmed change utxo (if any) to UtxoStore. + if changeUtxo != nil { + w.UtxoStore.s = append(w.UtxoStore.s, changeUtxo) + w.ReqSpentUtxoNtfn(changeUtxo) + modified = true + } + if modified { w.UtxoStore.dirty = true w.UtxoStore.Unlock() @@ -607,7 +615,7 @@ func SendMany(reply chan []byte, msg *btcjson.Message) { TxFee.Lock() fee := TxFee.i TxFee.Unlock() - rawtx, inputs, err := w.txToPairs(pairs, uint64(fee), int(minconf)) + rawtx, inputs, changeUtxo, err := w.txToPairs(pairs, uint64(fee), int(minconf)) if err != nil { e := InternalError e.Message = err.Error() @@ -642,6 +650,14 @@ func SendMany(reply chan []byte, msg *btcjson.Message) { // Remove previous unspent outputs now spent by the tx. w.UtxoStore.Lock() modified := w.UtxoStore.s.Remove(inputs) + + // Add unconfirmed change utxo (if any) to UtxoStore. + if changeUtxo != nil { + w.UtxoStore.s = append(w.UtxoStore.s, changeUtxo) + w.ReqSpentUtxoNtfn(changeUtxo) + modified = true + } + if modified { w.UtxoStore.dirty = true w.UtxoStore.Unlock() diff --git a/createtx.go b/createtx.go index 42b4801..6d143ea 100644 --- a/createtx.go +++ b/createtx.go @@ -101,9 +101,11 @@ func selectInputs(s tx.UtxoStore, amt uint64, minconf int) (inputs []*tx.Utxo, b // specifies the minimum number of confirmations required before an // unspent output is eligible for spending. Leftover input funds not sent // to addr or as a fee for the miner are sent to a newly generated -// address. ErrInsufficientFunds is returned if there are not enough -// eligible unspent outputs to create the transaction. -func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) (rawtx []byte, inputs []*tx.Utxo, err error) { +// address. If change is needed to return funds back to an owned +// address, changeUtxo will point to a unconfirmed (height = -1, zeroed +// block hash) Utxo. ErrInsufficientFunds is returned if there are not +// enough eligible unspent outputs to create the transaction. +func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) (rawtx []byte, inputs []*tx.Utxo, changeUtxo *tx.Utxo, err error) { // Recorded unspent transactions should not be modified until this // finishes. w.UtxoStore.RLock() @@ -121,20 +123,20 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) // Select unspent outputs to be used in transaction. inputs, btcout, err := selectInputs(w.UtxoStore.s, amt+fee, minconf) if err != nil { - return nil, nil, err + return nil, nil, nil, err } // Add outputs to new tx. for addr, amt := range pairs { addr160, _, err := btcutil.DecodeAddress(addr) if err != nil { - return nil, nil, fmt.Errorf("cannot decode address: %s", err) + return nil, nil, nil, fmt.Errorf("cannot decode address: %s", err) } // Spend amt to addr160 pkScript, err := btcscript.PayToPubKeyHashScript(addr160) if err != nil { - return nil, nil, fmt.Errorf("cannot create txout script: %s", err) + return nil, nil, nil, fmt.Errorf("cannot create txout script: %s", err) } txout := btcwire.NewTxOut(int64(amt), pkScript) msgtx.AddTxOut(txout) @@ -147,20 +149,33 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) // TODO(jrick): use the next chained address, not the next unused. newaddr, err := w.NextUnusedAddress() if err != nil { - return nil, nil, fmt.Errorf("failed to get next unused address: %s", err) + return nil, nil, nil, fmt.Errorf("failed to get next unused address: %s", err) } // Spend change change := btcout - (amt + fee) newaddr160, _, err := btcutil.DecodeAddress(newaddr) if err != nil { - return nil, nil, fmt.Errorf("cannot decode new address: %s", err) + return nil, nil, nil, fmt.Errorf("cannot decode new address: %s", err) } pkScript, err := btcscript.PayToPubKeyHashScript(newaddr160) if err != nil { - return nil, nil, fmt.Errorf("cannot create txout script: %s", err) + return nil, nil, nil, fmt.Errorf("cannot create txout script: %s", err) } msgtx.AddTxOut(btcwire.NewTxOut(int64(change), pkScript)) + + changeUtxo = &tx.Utxo{ + Amt: change, + Out: tx.OutPoint{ + // Hash is unset (zeroed) here and must be filled in + // with the transaction hash of the complete + // transaction. + Index: uint32(len(pairs)), + }, + Height: -1, + Subscript: pkScript, + } + copy(changeUtxo.AddrHash[:], newaddr160) } // Selected unspent outputs become new transaction's inputs. @@ -170,11 +185,11 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) for i, ip := range inputs { addrstr, err := btcutil.EncodeAddress(ip.AddrHash[:], w.Wallet.Net()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } privkey, err := w.GetAddressKey(addrstr) if err != nil { - return nil, nil, fmt.Errorf("cannot get address key: %v", err) + return nil, nil, nil, fmt.Errorf("cannot get address key: %v", err) } // TODO(jrick): we want compressed pubkeys. Switch wallet to @@ -183,7 +198,7 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) sigscript, err := btcscript.SignatureScript(msgtx, i, ip.Subscript, btcscript.SigHashAll, privkey, false) if err != nil { - return nil, nil, fmt.Errorf("cannot create sigscript: %s", err) + return nil, nil, nil, fmt.Errorf("cannot create sigscript: %s", err) } msgtx.TxIn[i].SignatureScript = sigscript } @@ -194,14 +209,23 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) engine, err := btcscript.NewScript(txin.SignatureScript, inputs[i].Subscript, i, msgtx, bip16) if err != nil { - return nil, nil, fmt.Errorf("cannot create script engine: %s", err) + return nil, nil, nil, fmt.Errorf("cannot create script engine: %s", err) } if err = engine.Execute(); err != nil { - return nil, nil, fmt.Errorf("cannot validate transaction: %s", err) + return nil, nil, nil, fmt.Errorf("cannot validate transaction: %s", err) } } + // Fill Tx hash of change outpoint with transaction hash. + if changeUtxo != nil { + txHash, err := msgtx.TxSha() + if err != nil { + return nil, nil, nil, fmt.Errorf("cannot create transaction hash: %s", err) + } + copy(changeUtxo.Out.Hash[:], txHash[:]) + } + buf := new(bytes.Buffer) msgtx.BtcEncode(buf, btcwire.ProtocolVersion) - return buf.Bytes(), inputs, nil + return buf.Bytes(), inputs, changeUtxo, nil } diff --git a/tx/tx.go b/tx/tx.go index 312d8c0..431f823 100644 --- a/tx/tx.go +++ b/tx/tx.go @@ -43,7 +43,11 @@ type Utxo struct { Out OutPoint Subscript PkScript Amt uint64 // Measured in Satoshis + + // Height is -1 if Utxo has not yet appeared in a block. Height int64 + + // BlockHash is zeroed if Utxo has not yet appeared in a block. BlockHash btcwire.ShaHash }