Perform smarter UTXO tracking.

This change fixes many issues with the tracking of unspent transaction
outputs.  First, notifications for when UTXOs arse spent are now
requested from btcd, and when spent, will be removed from the
UtxoStore.

Second, when transactions are created, the unconfirmed (not yet in a
block) Utxo (with block height -1 and zeroed block hash) is added to
the wallet's UtxoStore.  Notifications for when this UTXO is spent are
also requested from btcd.  After the tx appears in a block, because
the UTXO has a pkScript to be spent by another owned wallet address, a
notification with the UTXO will be sent to btcwallet.  We already
store the unconfirmed UTXO, so at this point the actual block height
and hash are filled in.

Finally, when calculating the balance, if confirmations is zero,
unconfirmed UTXOs (block height -1) will be included in the balance.
Otherwise, they are ignored.

cmd.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -63,6 +64,7 @@ type BtcWallet struct {
 	name              string
 	dirty             bool
 	NewBlockTxSeqN    uint64
+	SpentOutpointSeqN uint64
 	UtxoStore         struct {
 		sync.RWMutex
 		dirty bool
@@ -276,9 +278,13 @@ func (w *BtcWallet) CalculateBalance(confirmations int) float64 {
 	w.UtxoStore.RLock()
 	for _, u := range w.UtxoStore.s {
 		if int(height-u.Height) >= confirmations {
+			// Utxos not yet in blocks (height -1) should only be
+			// added if confirmations is 0.
+			if u.Height != -1 || (confirmations == 0 && u.Height == -1) {
 				bal += u.Amt
 			}
 		}
+	}
 	w.UtxoStore.RUnlock()
 	return float64(bal) / satoshiPerBTC
 }
@@ -298,6 +304,20 @@ func (w *BtcWallet) Track() {
 	for _, addr := range w.GetActiveAddresses() {
 		w.ReqNewTxsForAddress(addr)
 	}
+
+	n = <-NewJSONID
+	w.mtx.Lock()
+	w.SpentOutpointSeqN = n
+	w.mtx.Unlock()
+
+	replyHandlers.Lock()
+	replyHandlers.m[n] = w.spentUtxoHandler
+	replyHandlers.Unlock()
+	w.UtxoStore.RLock()
+	for _, utxo := range w.UtxoStore.s {
+		w.ReqSpentUtxoNtfn(utxo)
+	}
+	w.UtxoStore.RUnlock()
 }
 
 // RescanForAddress requests btcd to rescan the blockchain for new
@@ -358,6 +378,63 @@ func (w *BtcWallet) ReqNewTxsForAddress(addr string) {
 	btcdMsgs <- msg
 }
 
+// ReqSpentUtxoNtfn sends a message to btcd to request updates for when
+// a stored UTXO has been spent.
+func (w *BtcWallet) ReqSpentUtxoNtfn(u *tx.Utxo) {
+	log.Debugf("Requesting spent UTXO notifications for Outpoint hash %s index %d",
+		u.Out.Hash, u.Out.Index)
+
+	w.mtx.RLock()
+	n := w.SpentOutpointSeqN
+	w.mtx.RUnlock()
+
+	m := &btcjson.Message{
+		Jsonrpc: "1.0",
+		Id:      fmt.Sprintf("btcwallet(%d)", n),
+		Method:  "notifyspent",
+		Params: []interface{}{
+			u.Out.Hash.String(),
+			u.Out.Index,
+		},
+	}
+	msg, _ := json.Marshal(m)
+
+	btcdMsgs <- msg
+}
+
+// spentUtxoHandler is the handler function for btcd spent UTXO notifications
+// resulting from transactions in newly-attached blocks.
+func (w *BtcWallet) spentUtxoHandler(result interface{}, e *btcjson.Error) bool {
+	if e != nil {
+		log.Errorf("Spent UTXO Handler: Error %d received from btcd: %s",
+			e.Code, e.Message)
+		return false
+	}
+	v, ok := result.(map[string]interface{})
+	if !ok {
+		return false
+	}
+	txHashBE, ok := v["txhash"].(string)
+	if !ok {
+		log.Error("Spent UTXO Handler: Unspecified transaction hash.")
+		return false
+	}
+	txHash, err := btcwire.NewShaHashFromStr(txHashBE)
+	if err != nil {
+		log.Errorf("Spent UTXO Handler: Bad transaction hash: %s", err)
+		return false
+	}
+	index, ok := v["index"].(float64)
+	if !ok {
+		log.Error("Spent UTXO Handler: Unspecified index.")
+	}
+
+	_, _ = txHash, index
+
+	// Never remove this handler.
+	return false
+}
+
 // newBlockTxHandler is the handler function for btcd transaction
 // notifications resulting from newly-attached blocks.
 func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool {
@@ -467,6 +544,25 @@ func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool
 	// Do not add output to utxo store if spent.
 	if !spent {
 		go func() {
+			// First, iterate through all stored utxos.  If an unconfirmed utxo
+			// (not present in a block) has the same outpoint as this utxo,
+			// update the block height and hash.
+			w.UtxoStore.RLock()
+			for _, u := range w.UtxoStore.s {
+				if u.Height != -1 {
+					continue
+				}
+				if bytes.Equal(u.Out.Hash[:], txhash[:]) && u.Out.Index == uint32(index) {
+					// Found it.
+					fmt.Println("omg everything worked.")
+					copy(u.BlockHash[:], blockhash[:])
+					u.Height = int64(height)
+					w.UtxoStore.RUnlock()
+					return
+				}
+			}
+			w.UtxoStore.RUnlock()
+
 			u := &tx.Utxo{
 				Amt:       uint64(amt),
 				Height:    int64(height),

cmdmgr.go

@@ -460,7 +460,7 @@ func SendFrom(reply chan []byte, msg *btcjson.Message) {
 	pairs := map[string]uint64{
 		toaddr58: uint64(amt),
 	}
-	rawtx, inputs, err := w.txToPairs(pairs, uint64(fee), int(minconf))
+	rawtx, inputs, changeUtxo, err := w.txToPairs(pairs, uint64(fee), int(minconf))
 	if err != nil {
 		e := InternalError
 		e.Message = err.Error()
@@ -495,6 +495,14 @@ func SendFrom(reply chan []byte, msg *btcjson.Message) {
 		// Remove previous unspent outputs now spent by the tx.
 		w.UtxoStore.Lock()
 		modified := w.UtxoStore.s.Remove(inputs)
+
+		// Add unconfirmed change utxo (if any) to UtxoStore.
+		if changeUtxo != nil {
+			w.UtxoStore.s = append(w.UtxoStore.s, changeUtxo)
+			w.ReqSpentUtxoNtfn(changeUtxo)
+			modified = true
+		}
+
 		if modified {
 			w.UtxoStore.dirty = true
 			w.UtxoStore.Unlock()
@@ -607,7 +615,7 @@ func SendMany(reply chan []byte, msg *btcjson.Message) {
 	TxFee.Lock()
 	fee := TxFee.i
 	TxFee.Unlock()
-	rawtx, inputs, err := w.txToPairs(pairs, uint64(fee), int(minconf))
+	rawtx, inputs, changeUtxo, err := w.txToPairs(pairs, uint64(fee), int(minconf))
 	if err != nil {
 		e := InternalError
 		e.Message = err.Error()
@@ -642,6 +650,14 @@ func SendMany(reply chan []byte, msg *btcjson.Message) {
 		// Remove previous unspent outputs now spent by the tx.
 		w.UtxoStore.Lock()
 		modified := w.UtxoStore.s.Remove(inputs)
+
+		// Add unconfirmed change utxo (if any) to UtxoStore.
+		if changeUtxo != nil {
+			w.UtxoStore.s = append(w.UtxoStore.s, changeUtxo)
+			w.ReqSpentUtxoNtfn(changeUtxo)
+			modified = true
+		}
+
 		if modified {
 			w.UtxoStore.dirty = true
 			w.UtxoStore.Unlock()

createtx.go

@@ -101,9 +101,11 @@ func selectInputs(s tx.UtxoStore, amt uint64, minconf int) (inputs []*tx.Utxo, b
 // specifies the minimum number of confirmations required before an
 // unspent output is eligible for spending. Leftover input funds not sent
 // to addr or as a fee for the miner are sent to a newly generated
-// address. ErrInsufficientFunds is returned if there are not enough
-// eligible unspent outputs to create the transaction.
-func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) (rawtx []byte, inputs []*tx.Utxo, err error) {
+// address. If change is needed to return funds back to an owned
+// address, changeUtxo will point to a unconfirmed (height = -1, zeroed
+// block hash) Utxo.  ErrInsufficientFunds is returned if there are not
+// enough eligible unspent outputs to create the transaction.
+func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int) (rawtx []byte, inputs []*tx.Utxo, changeUtxo *tx.Utxo, err error) {
 	// Recorded unspent transactions should not be modified until this
 	// finishes.
 	w.UtxoStore.RLock()
@@ -121,20 +123,20 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int)
 	// Select unspent outputs to be used in transaction.
 	inputs, btcout, err := selectInputs(w.UtxoStore.s, amt+fee, minconf)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	// Add outputs to new tx.
 	for addr, amt := range pairs {
 		addr160, _, err := btcutil.DecodeAddress(addr)
 		if err != nil {
-			return nil, nil, fmt.Errorf("cannot decode address: %s", err)
+			return nil, nil, nil, fmt.Errorf("cannot decode address: %s", err)
 		}
 
 		// Spend amt to addr160
 		pkScript, err := btcscript.PayToPubKeyHashScript(addr160)
 		if err != nil {
-			return nil, nil, fmt.Errorf("cannot create txout script: %s", err)
+			return nil, nil, nil, fmt.Errorf("cannot create txout script: %s", err)
 		}
 		txout := btcwire.NewTxOut(int64(amt), pkScript)
 		msgtx.AddTxOut(txout)
@@ -147,20 +149,33 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int)
 		// TODO(jrick): use the next chained address, not the next unused.
 		newaddr, err := w.NextUnusedAddress()
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed to get next unused address: %s", err)
+			return nil, nil, nil, fmt.Errorf("failed to get next unused address: %s", err)
 		}
 
 		// Spend change
 		change := btcout - (amt + fee)
 		newaddr160, _, err := btcutil.DecodeAddress(newaddr)
 		if err != nil {
-			return nil, nil, fmt.Errorf("cannot decode new address: %s", err)
+			return nil, nil, nil, fmt.Errorf("cannot decode new address: %s", err)
 		}
 		pkScript, err := btcscript.PayToPubKeyHashScript(newaddr160)
 		if err != nil {
-			return nil, nil, fmt.Errorf("cannot create txout script: %s", err)
+			return nil, nil, nil, fmt.Errorf("cannot create txout script: %s", err)
 		}
 		msgtx.AddTxOut(btcwire.NewTxOut(int64(change), pkScript))
+
+		changeUtxo = &tx.Utxo{
+			Amt: change,
+			Out: tx.OutPoint{
+				// Hash is unset (zeroed) here and must be filled in
+				// with the transaction hash of the complete
+				// transaction.
+				Index: uint32(len(pairs)),
+			},
+			Height:    -1,
+			Subscript: pkScript,
+		}
+		copy(changeUtxo.AddrHash[:], newaddr160)
 	}
 
 	// Selected unspent outputs become new transaction's inputs.
@@ -170,11 +185,11 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int)
 	for i, ip := range inputs {
 		addrstr, err := btcutil.EncodeAddress(ip.AddrHash[:], w.Wallet.Net())
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		privkey, err := w.GetAddressKey(addrstr)
 		if err != nil {
-			return nil, nil, fmt.Errorf("cannot get address key: %v", err)
+			return nil, nil, nil, fmt.Errorf("cannot get address key: %v", err)
 		}
 
 		// TODO(jrick): we want compressed pubkeys.  Switch wallet to
@@ -183,7 +198,7 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int)
 		sigscript, err := btcscript.SignatureScript(msgtx, i,
 			ip.Subscript, btcscript.SigHashAll, privkey, false)
 		if err != nil {
-			return nil, nil, fmt.Errorf("cannot create sigscript: %s", err)
+			return nil, nil, nil, fmt.Errorf("cannot create sigscript: %s", err)
 		}
 		msgtx.TxIn[i].SignatureScript = sigscript
 	}
@@ -194,14 +209,23 @@ func (w *BtcWallet) txToPairs(pairs map[string]uint64, fee uint64, minconf int)
 		engine, err := btcscript.NewScript(txin.SignatureScript, inputs[i].Subscript, i,
 			msgtx, bip16)
 		if err != nil {
-			return nil, nil, fmt.Errorf("cannot create script engine: %s", err)
+			return nil, nil, nil, fmt.Errorf("cannot create script engine: %s", err)
 		}
 		if err = engine.Execute(); err != nil {
-			return nil, nil, fmt.Errorf("cannot validate transaction: %s", err)
+			return nil, nil, nil, fmt.Errorf("cannot validate transaction: %s", err)
 		}
 	}
 
+	// Fill Tx hash of change outpoint with transaction hash.
+	if changeUtxo != nil {
+		txHash, err := msgtx.TxSha()
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("cannot create transaction hash: %s", err)
+		}
+		copy(changeUtxo.Out.Hash[:], txHash[:])
+	}
+
 	buf := new(bytes.Buffer)
 	msgtx.BtcEncode(buf, btcwire.ProtocolVersion)
-	return buf.Bytes(), inputs, nil
+	return buf.Bytes(), inputs, changeUtxo, nil
 }

tx/tx.go

@@ -43,7 +43,11 @@ type Utxo struct {
 	Out       OutPoint
 	Subscript PkScript
 	Amt       uint64 // Measured in Satoshis
+
+	// Height is -1 if Utxo has not yet appeared in a block.
 	Height    int64
+
+	// BlockHash is zeroed if Utxo has not yet appeared in a block.
 	BlockHash btcwire.ShaHash
 }