Implement address rescanning.

When a wallet is opened, a rescan request will be sent to btcd with
all active addresses from the wallet, to rescan from the last synced
block (now saved to the wallet file) and the current best block.

As multi-account support is further explored, rescan requests should
be batched together to send a single request for all addresses from
all wallets.

This change introduces several changes to the wallet, tx, and utxo
files.  Wallet files are still compatible, however, a rescan will try
to start at the genesis block since no correct "last synced to" or
"created at block X" was saved.  The tx and utxo files, however, are
not compatible and should be deleted (or an error will occur on read).
If any errors occur opening the utxo file, a rescan will start
beginning at the creation block saved in the wallet.

cmd.go

@@ -32,23 +32,32 @@ import (
 	"time"
 )
 
-const (
-	satoshiPerBTC = 100000000
-)
-
 var (
 	// ErrNoWallet describes an error where a wallet does not exist and
 	// must be created first.
 	ErrNoWallet = errors.New("wallet file does not exist")
 
+	// ErrNoUtxos describes an error where the wallet file was successfully
+	// read, but the UTXO file was not.  To properly handle this error,
+	// a rescan should be done since the wallet creation block.
+	ErrNoUtxos = errors.New("utxo file cannot be read")
+
+	// ErrNoTxs describes an error where the wallet and UTXO files were
+	// successfully read, but the TX history file was not.  It is up to
+	// the caller whether this necessitates a rescan or not.
+	ErrNoTxs = errors.New("tx file cannot be read")
+
 	cfg *config
 
-	curHeight = struct {
+	curBlock = struct {
 		sync.RWMutex
-		h int64
+		wallet.BlockStamp
 	}{
-		h: btcutil.BlockHeightUnknown,
+		BlockStamp: wallet.BlockStamp{
+			Height: int32(btcutil.BlockHeightUnknown),
+		},
 	}
+
 	wallets = NewBtcWalletStore()
 )
 
@@ -61,6 +70,7 @@ type BtcWallet struct {
 	mtx               sync.RWMutex
 	name              string
 	dirty             bool
+	fullRescan        bool
 	NewBlockTxSeqN    uint64
 	SpentOutpointSeqN uint64
 	UtxoStore         struct {
@@ -95,7 +105,7 @@ func NewBtcWalletStore() *BtcWalletStore {
 //
 // TODO(jrick): This must also roll back the UTXO and TX stores, and notify
 // all wallets of new account balances.
-func (s *BtcWalletStore) Rollback(height int64, hash *btcwire.ShaHash) {
+func (s *BtcWalletStore) Rollback(height int32, hash *btcwire.ShaHash) {
 	for _, w := range s.m {
 		w.Rollback(height, hash)
 	}
@@ -105,7 +115,7 @@ func (s *BtcWalletStore) Rollback(height int64, hash *btcwire.ShaHash) {
 // with the passed chainheight and block hash was connected to the main
 // chain.  This is used to remove transactions and utxos for each wallet
 // that occured on a chain no longer considered to be the main chain.
-func (w *BtcWallet) Rollback(height int64, hash *btcwire.ShaHash) {
+func (w *BtcWallet) Rollback(height int32, hash *btcwire.ShaHash) {
 	w.UtxoStore.Lock()
 	w.UtxoStore.dirty = w.UtxoStore.dirty || w.UtxoStore.s.Rollback(height, hash)
 	w.UtxoStore.Unlock()
@@ -157,9 +167,11 @@ func OpenWallet(cfg *config, account string) (*BtcWallet, error) {
 	}
 
 	wfilepath := filepath.Join(wdir, "wallet.bin")
-	txfilepath := filepath.Join(wdir, "tx.bin")
 	utxofilepath := filepath.Join(wdir, "utxo.bin")
-	var wfile, txfile, utxofile *os.File
+	txfilepath := filepath.Join(wdir, "tx.bin")
+	var wfile, utxofile, txfile *os.File
+
+	// Read wallet file.
 	if wfile, err = os.Open(wfilepath); err != nil {
 		if os.IsNotExist(err) {
 			// Must create and save wallet first.
@@ -168,80 +180,114 @@ func OpenWallet(cfg *config, account string) (*BtcWallet, error) {
 		return nil, fmt.Errorf("cannot open wallet file: %s", err)
 	}
 	defer wfile.Close()
-	if txfile, err = os.Open(txfilepath); err != nil {
-		if os.IsNotExist(err) {
-			if txfile, err = os.Create(txfilepath); err != nil {
-				return nil, fmt.Errorf("cannot create tx file: %s", err)
-			}
-		} else {
-			return nil, fmt.Errorf("cannot open tx file: %s", err)
-		}
-	}
-	defer txfile.Close()
-	if utxofile, err = os.Open(utxofilepath); err != nil {
-		if os.IsNotExist(err) {
-			if utxofile, err = os.Create(utxofilepath); err != nil {
-				return nil, fmt.Errorf("cannot create utxo file: %s", err)
-			}
-		} else {
-			return nil, fmt.Errorf("cannot open utxo file: %s", err)
-		}
-	}
-	defer utxofile.Close()
 
 	wlt := new(wallet.Wallet)
 	if _, err = wlt.ReadFrom(wfile); err != nil {
 		return nil, fmt.Errorf("cannot read wallet: %s", err)
 	}
 
-	var txs tx.TxStore
-	if _, err = txs.ReadFrom(txfile); err != nil {
-		return nil, fmt.Errorf("cannot read tx file: %s", err)
-	}
-
-	var utxos tx.UtxoStore
-	if _, err = utxos.ReadFrom(utxofile); err != nil {
-		return nil, fmt.Errorf("cannot read utxo file: %s", err)
-	}
-
 	w := &BtcWallet{
 		Wallet: wlt,
 		name:   account,
 	}
+
+	// Read utxo file.  If this fails, return a ErrNoUtxos error so a
+	// rescan can be done since the wallet creation block.
+	var utxos tx.UtxoStore
+	if utxofile, err = os.Open(utxofilepath); err != nil {
+		log.Errorf("cannot open utxo file: %s", err)
+		return w, ErrNoUtxos
+	}
+	defer utxofile.Close()
+	if _, err = utxos.ReadFrom(utxofile); err != nil {
+		log.Errorf("cannot read utxo file: %s", err)
+		return w, ErrNoUtxos
+	}
 	w.UtxoStore.s = utxos
+
+	// Read tx file.  If this fails, return a ErrNoTxs error and let
+	// the caller decide if a rescan is necessary.
+	if txfile, err = os.Open(txfilepath); err != nil {
+		log.Errorf("cannot open tx file: %s", err)
+		return w, ErrNoTxs
+	}
+	defer txfile.Close()
+	var txs tx.TxStore
+	if _, err = txs.ReadFrom(txfile); err != nil {
+		log.Errorf("cannot read tx file: %s", err)
+		return w, ErrNoTxs
+	}
 	w.TxStore.s = txs
 
 	return w, nil
 }
 
-func getCurHeight() (height int64) {
+// GetCurBlock returns the blockchain height and SHA hash of the most
-	curHeight.RLock()
+// recently seen block.  If no blocks have been seen since btcd has
-	height = curHeight.h
+// connected, btcd is queried for the current block height and hash.
-	curHeight.RUnlock()
+func GetCurBlock() (bs wallet.BlockStamp, err error) {
-	if height != btcutil.BlockHeightUnknown {
+	curBlock.RLock()
-		return height
+	bs = curBlock.BlockStamp
+	curBlock.RUnlock()
+	if bs.Height != int32(btcutil.BlockHeightUnknown) {
+		return bs, nil
+	}
+
+	// This is a hack and may result in races, but we need to make
+	// sure that btcd is connected and sending a message will succeed,
+	// or this will block forever. A better solution is to return an
+	// error to the reply handler immediately if btcd is disconnected.
+	if !btcdConnected.b {
+		return wallet.BlockStamp{
+			Height: int32(btcutil.BlockHeightUnknown),
+		}, errors.New("current block unavailable")
 	}
 
 	n := <-NewJSONID
-	m, err := btcjson.CreateMessageWithId("getblockcount",
+	msg := btcjson.Message{
-		fmt.Sprintf("btcwallet(%v)", n))
+		Jsonrpc: "1.0",
-	if err != nil {
+		Id:      fmt.Sprintf("btcwallet(%v)", n),
-		// Can't continue.
+		Method:  "getbestblock",
-		return btcutil.BlockHeightUnknown
 	}
+	m, _ := json.Marshal(msg)
 
-	c := make(chan int64)
+	c := make(chan *struct {
+		hash   *btcwire.ShaHash
+		height int32
+	})
 
 	replyHandlers.Lock()
 	replyHandlers.m[n] = func(result interface{}, e *btcjson.Error) bool {
 		if e != nil {
-			c <- btcutil.BlockHeightUnknown
+			c <- nil
 			return true
 		}
-		if balance, ok := result.(float64); ok {
+		m, ok := result.(map[string]interface{})
-			c <- int64(balance)
+		if !ok {
-		} else {
+			c <- nil
-			c <- btcutil.BlockHeightUnknown
+			return true
+		}
+		hashBE, ok := m["hash"].(string)
+		if !ok {
+			c <- nil
+			return true
+		}
+		hash, err := btcwire.NewShaHashFromStr(hashBE)
+		if err != nil {
+			c <- nil
+			return true
+		}
+		fheight, ok := m["height"].(float64)
+		if !ok {
+			c <- nil
+			return true
+		}
+		c <- &struct {
+			hash   *btcwire.ShaHash
+			height int32
+		}{
+			hash:   hash,
+			height: int32(fheight),
 		}
 		return true
 	}
@@ -251,16 +297,22 @@ func getCurHeight() (height int64) {
 	btcdMsgs <- m
 
 	// Block until reply is ready.
-	height = <-c
+	if reply := <-c; reply != nil {
-	curHeight.Lock()
+		curBlock.Lock()
-	if height > curHeight.h {
+		if reply.height > curBlock.BlockStamp.Height {
-		curHeight.h = height
+			bs = wallet.BlockStamp{
-	} else {
+				Height: reply.height,
-		height = curHeight.h
+				Hash:   *reply.hash,
+			}
+			curBlock.BlockStamp = bs
+		}
+		curBlock.Unlock()
+		return bs, nil
 	}
-	curHeight.Unlock()
 
-	return height
+	return wallet.BlockStamp{
+		Height: int32(btcutil.BlockHeightUnknown),
+	}, errors.New("current block unavailable")
 }
 
 // CalculateBalance sums the amounts of all unspent transaction
@@ -275,8 +327,8 @@ func getCurHeight() (height int64) {
 func (w *BtcWallet) CalculateBalance(confirms int) float64 {
 	var bal uint64 // Measured in satoshi
 
-	height := getCurHeight()
+	bs, err := GetCurBlock()
-	if height == btcutil.BlockHeightUnknown {
+	if bs.Height == int32(btcutil.BlockHeightUnknown) || err != nil {
 		return 0.
 	}
 
@@ -284,12 +336,12 @@ func (w *BtcWallet) CalculateBalance(confirms int) float64 {
 	for _, u := range w.UtxoStore.s {
 		// Utxos not yet in blocks (height -1) should only be
 		// added if confirmations is 0.
-		if confirms == 0 || (u.Height != -1 && int(height-u.Height+1) >= confirms) {
+		if confirms == 0 || (u.Height != -1 && int(bs.Height-u.Height+1) >= confirms) {
 			bal += u.Amt
 		}
 	}
 	w.UtxoStore.RUnlock()
-	return float64(bal) / satoshiPerBTC
+	return float64(bal) / float64(btcutil.SatoshiPerBitcoin)
 }
 
 // Track requests btcd to send notifications of new transactions for
@@ -305,7 +357,7 @@ func (w *BtcWallet) Track() {
 	replyHandlers.m[n] = w.newBlockTxHandler
 	replyHandlers.Unlock()
 	for _, addr := range w.GetActiveAddresses() {
-		w.ReqNewTxsForAddress(addr)
+		w.ReqNewTxsForAddress(addr.Address)
 	}
 
 	n = <-NewJSONID
@@ -323,21 +375,36 @@ func (w *BtcWallet) Track() {
 	w.UtxoStore.RUnlock()
 }
 
-// RescanForAddress requests btcd to rescan the blockchain for new
+// RescanToBestBlock requests btcd to rescan the blockchain for new
-// transactions to addr.  This is useful for making btcwallet catch up to
+// transactions to all wallet addresses.  This is needed for making
-// a long-running btcd process, or for importing addresses and rescanning
+// btcwallet catch up to a long-running btcd process, as otherwise
-// for unspent tx outputs.  If len(blocks) is 0, the entire blockchain is
+// it would have missed notifications as blocks are attached to the
-// rescanned.  If len(blocks) is 1, the rescan will begin at height
+// main chain.
-// blocks[0].  If len(blocks) is 2 or greater, the rescan will be
+func (w *BtcWallet) RescanToBestBlock() {
-// performed for the block range blocks[0]...blocks[1] (inclusive).
+	beginBlock := int32(0)
-func (w *BtcWallet) RescanForAddress(addr string, blocks ...int) {
+
-	n := <-NewJSONID
+	if w.fullRescan {
-	params := []interface{}{addr}
+		// Need to perform a complete rescan since the wallet creation
-	if len(blocks) > 0 {
+		// block.
-		params = append(params, blocks[0])
+		beginBlock = w.CreatedAt()
+		log.Debugf("Rescanning account '%v' for new transactions since block height %v",
+			w.name, beginBlock)
+	} else {
+		// The last synced block height should be used the starting
+		// point for block rescanning.  Grab the block stamp here.
+		bs := w.SyncedWith()
+
+		log.Debugf("Rescanning account '%v' for new transactions since block height %v hash %v",
+			w.name, bs.Height, bs.Hash)
+
+		// If we're synced with block x, must scan the blocks x+1 to best block.
+		beginBlock = bs.Height + 1
 	}
-	if len(blocks) > 1 {
+
-		params = append(params, blocks[1])
+	n := <-NewJSONID
+	params := []interface{}{
+		beginBlock,
+		w.ActivePaymentAddresses(),
 	}
 	m := &btcjson.Message{
 		Jsonrpc: "1.0",
@@ -349,18 +416,53 @@ func (w *BtcWallet) RescanForAddress(addr string, blocks ...int) {
 
 	replyHandlers.Lock()
 	replyHandlers.m[n] = func(result interface{}, e *btcjson.Error) bool {
-		// TODO(jrick)
+		// Rescan is compatible with new txs from connected block
+		// notifications, so use that handler.
+		_ = w.newBlockTxHandler(result, e)
 
-		// btcd returns a nil result when the rescan is complete.
+		if result != nil {
-		// Returning true signals that this handler is finished
+			// Notify frontends of new account balance.
-		// and can be removed.
+			confirmed := w.CalculateBalance(1)
-		return result == nil
+			unconfirmed := w.CalculateBalance(0) - confirmed
+			NotifyWalletBalance(frontendNotificationMaster, w.name, confirmed)
+			NotifyWalletBalanceUnconfirmed(frontendNotificationMaster, w.name, unconfirmed)
+
+			return false
+		}
+		if bs, err := GetCurBlock(); err == nil {
+			w.SetSyncedWith(&bs)
+			w.dirty = true
+			if err = w.writeDirtyToDisk(); err != nil {
+				log.Errorf("cannot sync dirty wallet: %v",
+					err)
+			}
+		}
+		// If result is nil, the rescan has completed.  Returning
+		// true removes this handler.
+		return true
 	}
 	replyHandlers.Unlock()
 
 	btcdMsgs <- msg
 }
 
+// ActivePaymentAddresses returns the second parameter for all rescan
+// commands.  The returned slice  maps between payment address strings and
+// the block height to begin rescanning for transactions to that address.
+func (w *BtcWallet) ActivePaymentAddresses() []string {
+	w.mtx.RLock()
+	defer w.mtx.RUnlock()
+
+	infos := w.GetActiveAddresses()
+	addrs := make([]string, len(infos))
+
+	for i := range infos {
+		addrs[i] = infos[i].Address
+	}
+
+	return addrs
+}
+
 // ReqNewTxsForAddress sends a message to btcd to request tx updates
 // for addr for each new block that is added to the blockchain.
 func (w *BtcWallet) ReqNewTxsForAddress(addr string) {
@@ -550,16 +652,17 @@ func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool
 		// update the block height and hash.
 		w.UtxoStore.RLock()
 		for _, u := range w.UtxoStore.s {
-			if u.Height != -1 {
-				continue
-			}
 			if bytes.Equal(u.Out.Hash[:], txhash[:]) && u.Out.Index == uint32(index) {
-				// Found it.
+				// Found a either a duplicate, or a change UTXO.  If not change,
+				// ignore it.
+				if u.Height != -1 {
+					return false
+				}
 				w.UtxoStore.RUnlock()
 
 				w.UtxoStore.Lock()
 				copy(u.BlockHash[:], blockhash[:])
-				u.Height = int64(height)
+				u.Height = int32(height)
 				w.UtxoStore.dirty = true
 				w.UtxoStore.Unlock()
 
@@ -571,9 +674,12 @@ func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool
 		}
 		w.UtxoStore.RUnlock()
 
+		// After iterating through all UTXOs, it was not a duplicate or
+		// change UTXO appearing in a block.  Append a new Utxo to the end.
+
 		u := &tx.Utxo{
 			Amt:       uint64(amt),
-			Height:    int64(height),
+			Height:    int32(height),
 			Subscript: pkscript,
 		}
 		copy(u.Out.Hash[:], txhash[:])
@@ -638,14 +744,35 @@ func main() {
 
 	// Open default wallet
 	w, err := OpenWallet(cfg, "")
-	if err != nil {
+	switch err {
-		log.Info(err.Error())
+	case ErrNoTxs:
-	} else {
+		// Do nothing special for now.  This will be implemented when
+		// the tx history file is properly written.
 		wallets.Lock()
 		wallets.m[""] = w
 		wallets.Unlock()
+
+	case ErrNoUtxos:
+		// Add wallet, but mark wallet as needing a full rescan since
+		// the wallet creation block.  This will take place when btcd
+		// connects.
+		wallets.Lock()
+		wallets.m[""] = w
+		wallets.Unlock()
+		w.fullRescan = true
+
+	case nil:
+		wallets.Lock()
+		wallets.m[""] = w
+		wallets.Unlock()
+
+	default:
+		log.Errorf("cannot open wallet: %v", err)
 	}
 
+	// Start wallet disk syncer goroutine.
+	go DirtyWalletSyncer()
+
 	go func() {
 		// Start HTTP server to listen and send messages to frontend and btcd
 		// backend.  Try reconnection if connection failed.

cmdmgr.go

@@ -19,6 +19,7 @@ package main
 import (
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/conformal/btcjson"
 	"github.com/conformal/btcwallet/wallet"
@@ -26,6 +27,13 @@ import (
 	"time"
 )
 
+var (
+	// ErrBtcdDisconnected describes an error where an operation cannot
+	// successfully complete due to btcd not being connected to
+	// btcwallet.
+	ErrBtcdDisconnected = errors.New("btcd disconnected")
+)
+
 // ProcessFrontendMsg checks the message sent from a frontend.  If the
 // message method is one that must be handled by btcwallet, the request
 // is processed here.  Otherwise, the message is sent to btcd.
@@ -129,9 +137,9 @@ func GetAddressesByAccount(reply chan []byte, msg *btcjson.Message) {
 		return
 	}
 
-	var result interface{}
+	var result []string
 	if w := wallets.m[account]; w != nil {
-		result = w.Wallet.GetActiveAddresses()
+		result = w.ActivePaymentAddresses()
 	} else {
 		ReplyError(reply, msg.Id, &btcjson.ErrWalletInvalidAccountName)
 		return
@@ -689,7 +697,15 @@ func CreateEncryptedWallet(reply chan []byte, msg *btcjson.Message) {
 	} else {
 		net = btcwire.TestNet3
 	}
-	wlt, err := wallet.NewWallet(wname, desc, []byte(pass), net)
+
+	bs, err := GetCurBlock()
+	if err != nil {
+		e := btcjson.ErrInternal
+		e.Message = "btcd disconnected"
+		ReplyError(reply, msg.Id, &e)
+		return
+	}
+	wlt, err := wallet.NewWallet(wname, desc, []byte(pass), net, &bs)
 	if err != nil {
 		log.Error("Error creating wallet: " + err.Error())
 		ReplyError(reply, msg.Id, &btcjson.ErrInternal)

createtx.go

@@ -82,7 +82,10 @@ func (u ByAmount) Swap(i, j int) {
 // of all selected previous outputs.  err will equal ErrInsufficientFunds if there
 // are not enough unspent outputs to spend amt.
 func selectInputs(s tx.UtxoStore, amt uint64, minconf int) (inputs []*tx.Utxo, btcout uint64, err error) {
-	height := getCurHeight()
+	bs, err := GetCurBlock()
+	if err != nil {
+		return nil, 0, err
+	}
 
 	// Create list of eligible unspent previous outputs to use as tx
 	// inputs, and sort by the amount in reverse order so a minimum number
@@ -93,7 +96,7 @@ func selectInputs(s tx.UtxoStore, amt uint64, minconf int) (inputs []*tx.Utxo, b
 		// to a change address, resulting in a UTXO not yet mined in a block.
 		// For now, disallow creating transactions until these UTXOs are mined
 		// into a block and show up as part of the balance.
-		if utxo.Height != -1 && int(height-utxo.Height) >= minconf {
+		if utxo.Height != -1 && int(bs.Height-utxo.Height) >= minconf {
 			eligible = append(eligible, utxo)
 		}
 	}

createtx_test.go

@@ -1,10 +1,6 @@
 package main
 
 import (
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"github.com/conformal/btcjson"
 	"github.com/conformal/btcscript"
 	"github.com/conformal/btcutil"
 	"github.com/conformal/btcwallet/tx"
@@ -15,7 +11,8 @@ import (
 
 func TestFakeTxs(t *testing.T) {
 	// First we need a wallet.
-	w, err := wallet.NewWallet("banana wallet", "", []byte("banana"), btcwire.MainNet)
+	w, err := wallet.NewWallet("banana wallet", "", []byte("banana"),
+		btcwire.MainNet, &wallet.BlockStamp{})
 	if err != nil {
 		t.Errorf("Can not create encrypted wallet: %s", err)
 		return
@@ -58,30 +55,15 @@ func TestFakeTxs(t *testing.T) {
 	btcw.UtxoStore.s = append(btcw.UtxoStore.s, utxo)
 
 	// Fake our current block height so btcd doesn't need to be queried.
-	curHeight.h = 12346
+	curBlock.BlockStamp.Height = 12346
 
 	// Create the transaction.
 	pairs := map[string]uint64{
 		"17XhEvq9Nahdj7Xe1nv6oRe1tEmaHUuynH": 5000,
 	}
-	createdTx, err := btcw.txToPairs(pairs, 100, 0)
+	_, err = btcw.txToPairs(pairs, 100, 0)
 	if err != nil {
 		t.Errorf("Tx creation failed: %s", err)
 		return
 	}
-
-	msg := btcjson.Message{
-		Jsonrpc: "1.0",
-		Id:      "test",
-		Method:  "sendrawtransaction",
-		Params: []interface{}{
-			hex.EncodeToString(createdTx.rawTx),
-		},
-	}
-	m, _ := json.Marshal(msg)
-	_ = m
-	_ = fmt.Println
-
-	// Uncomment to print out json to send raw transaction
-	// fmt.Println(string(m))
 }

disksync.go

@@ -20,9 +20,47 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 )
 
+var (
+	// dirtyWallets holds a set of wallets that include dirty components.
+	dirtyWallets = struct {
+		sync.Mutex
+		m map[*BtcWallet]bool
+	}{
+		m: make(map[*BtcWallet]bool),
+	}
+)
+
+// DirtyWalletSyncer synces dirty wallets for cases where the updated
+// information was not required to be immediately written to disk.  Wallets
+// may be added to dirtyWallets and will be checked and processed every 10
+// seconds by this function.
+//
+// This never returns and is meant to be called from a goroutine.
+func DirtyWalletSyncer() {
+	ticker := time.Tick(10 * time.Second)
+	for {
+		select {
+		case <-ticker:
+			dirtyWallets.Lock()
+			for w := range dirtyWallets.m {
+				log.Debugf("Syncing wallet '%v' to disk",
+					w.Wallet.Name())
+				if err := w.writeDirtyToDisk(); err != nil {
+					log.Errorf("cannot sync dirty wallet: %v",
+						err)
+				} else {
+					delete(dirtyWallets.m, w)
+				}
+			}
+			dirtyWallets.Unlock()
+		}
+	}
+}
+
 // writeDirtyToDisk checks for the dirty flag on an account's wallet,
 // txstore, and utxostore, writing them to disk if any are dirty.
 func (w *BtcWallet) writeDirtyToDisk() error {

sockets.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/conformal/btcjson"
+	"github.com/conformal/btcwallet/wallet"
 	"github.com/conformal/btcwire"
 	"net"
 	"net/http"
@@ -110,8 +111,8 @@ func frontendListenerDuplicator() {
 				// place these notifications in that function.
 				NotifyBtcdConnected(frontendNotificationMaster,
 					btcdConnected.b)
-				if btcdConnected.b {
+				if bs, err := GetCurBlock(); err == nil {
-					NotifyNewBlockChainHeight(c, getCurHeight())
+					NotifyNewBlockChainHeight(c, bs.Height)
 					NotifyBalances(c)
 				}
 
@@ -144,6 +145,7 @@ func frontendListenerDuplicator() {
 	}
 }
 
+// NotifyBtcdConnected notifies all frontends of a new btcd connection.
 func NotifyBtcdConnected(reply chan []byte, conn bool) {
 	btcdConnected.b = conn
 	var idStr interface{} = "btcwallet:btcdconnected"
@@ -266,11 +268,17 @@ func ProcessBtcdNotificationReply(b []byte) {
 		log.Errorf("Unable to unmarshal btcd message: %v", err)
 		return
 	}
+	if r.Id == nil {
+		// btcd should only ever be sending JSON messages with a string in
+		// the id field.  Log the error and drop the message.
+		log.Error("Unable to process btcd notification or reply.")
+		return
+	}
 	idStr, ok := (*r.Id).(string)
 	if !ok {
 		// btcd should only ever be sending JSON messages with a string in
 		// the id field.  Log the error and drop the message.
-		log.Error("Unable to process btcd notification or reply.")
+		log.Error("Incorrect btcd notification id type.")
 		return
 	}
 
@@ -338,7 +346,7 @@ func ProcessBtcdNotificationReply(b []byte) {
 
 // NotifyNewBlockChainHeight notifies all frontends of a new
 // blockchain height.
-func NotifyNewBlockChainHeight(reply chan []byte, height int64) {
+func NotifyNewBlockChainHeight(reply chan []byte, height int32) {
 	var id interface{} = "btcwallet:newblockchainheight"
 	msgRaw := &btcjson.Reply{
 		Result: height,
@@ -372,7 +380,7 @@ func NtfnBlockConnected(r interface{}) {
 		log.Error("blockconnected notification: invalid height")
 		return
 	}
-	height := int64(heightf)
+	height := int32(heightf)
 	var minedTxs []string
 	if iminedTxs, ok := result["minedtxs"].([]interface{}); ok {
 		minedTxs = make([]string, len(iminedTxs))
@@ -386,12 +394,35 @@ func NtfnBlockConnected(r interface{}) {
 		}
 	}
 
-	curHeight.Lock()
+	curBlock.Lock()
-	curHeight.h = height
+	curBlock.BlockStamp = wallet.BlockStamp{
-	curHeight.Unlock()
+		Height: height,
+		Hash:   *hash,
+	}
+	curBlock.Unlock()
 
-	// TODO(jrick): update TxStore and UtxoStore with new hash
+	// btcd notifies btcwallet about transactions first, and then sends
-	_ = hash
+	// the block notification.  This prevents any races from saving a
+	// synced-to block before all notifications from the block have been
+	// processed.
+	bs := &wallet.BlockStamp{
+		Height: height,
+		Hash:   *hash,
+	}
+	for _, w := range wallets.m {
+		// We do not write synced info immediatelly out to disk.
+		// If btcd is performing an IBD, that would result in
+		// writing out the wallet to disk for each processed block.
+		// Instead, mark as dirty and let another goroutine process
+		// the dirty wallet.
+		w.mtx.Lock()
+		w.Wallet.SetSyncedWith(bs)
+		w.dirty = true
+		w.mtx.Unlock()
+		dirtyWallets.Lock()
+		dirtyWallets.m[w] = true
+		dirtyWallets.Unlock()
+	}
 
 	// Notify frontends of new blockchain height.
 	NotifyNewBlockChainHeight(frontendNotificationMaster, height)
@@ -451,7 +482,7 @@ func NtfnBlockDisconnected(r interface{}) {
 	if !ok {
 		log.Error("blockdisconnected notification: invalid height")
 	}
-	height := int64(heightf)
+	height := int32(heightf)
 
 	// Rollback Utxo and Tx data stores.
 	go func() {
@@ -571,21 +602,25 @@ func BtcdHandshake(ws *websocket.Conn) {
 		return
 	}
 
+	// TODO(jrick): Check that there was not any reorgs done
+	// since last connection.  If so, rollback and rescan to
+	// catch up.
+
+	for _, w := range wallets.m {
+		w.RescanToBestBlock()
+	}
+
 	// Begin tracking wallets against this btcd instance.
 	for _, w := range wallets.m {
 		w.Track()
 	}
 
-	// Request the new block height, and notify frontends.
-	//
-	// TODO(jrick): Check that there was not any reorgs done
-	// since last connection.
-	NotifyNewBlockChainHeight(frontendNotificationMaster, getCurHeight())
-
-	// Notify frontends of all account balances, calculated based
-	// from the block height of this new btcd connection.
-	NotifyBalances(frontendNotificationMaster)
-
 	// (Re)send any unmined transactions to btcd in case of a btcd restart.
 	resendUnminedTxs()
+
+	// Get current blockchain height and best block hash.
+	if bs, err := GetCurBlock(); err == nil {
+		NotifyNewBlockChainHeight(frontendNotificationMaster, bs.Height)
+		NotifyBalances(frontendNotificationMaster)
+	}
 }

tx/tx.go

@@ -45,7 +45,7 @@ type Utxo struct {
 	Amt       uint64 // Measured in Satoshis
 
 	// Height is -1 if Utxo has not yet appeared in a block.
-	Height    int64
+	Height int32
 
 	// BlockHash is zeroed if Utxo has not yet appeared in a block.
 	BlockHash btcwire.ShaHash
@@ -66,7 +66,7 @@ type TxStore []interface{}
 type RecvTx struct {
 	TxHash       btcwire.ShaHash
 	BlockHash    btcwire.ShaHash
-	Height       int64
+	Height       int32
 	Amt          uint64 // Measured in Satoshis
 	SenderAddr   [ripemd160.Size]byte
 	ReceiverAddr [ripemd160.Size]byte
@@ -77,7 +77,7 @@ type RecvTx struct {
 type SendTx struct {
 	TxHash        btcwire.ShaHash
 	BlockHash     btcwire.ShaHash
-	Height        int64
+	Height        int32
 	Fee           uint64 // Measured in Satoshis
 	SenderAddr    [ripemd160.Size]byte
 	ReceiverAddrs []struct {
@@ -156,7 +156,7 @@ func (u *UtxoStore) WriteTo(w io.Writer) (n int64, err error) {
 //
 // Correct results rely on u being sorted by block height in
 // increasing order.
-func (u *UtxoStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool) {
+func (u *UtxoStore) Rollback(height int32, hash *btcwire.ShaHash) (modified bool) {
 	s := *u
 
 	// endlen specifies the final length of the rolled-back UtxoStore.
@@ -219,7 +219,7 @@ func (u *UtxoStore) Remove(toRemove []*Utxo) (modified bool) {
 // ReadFrom satisifies the io.ReaderFrom interface.  A Utxo is read
 // from r with the format:
 //
-//  [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (8 bytes), BlockHash (32 bytes)]
+//  [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (4 bytes), BlockHash (32 bytes)]
 //
 // Each field is read little endian.
 func (u *Utxo) ReadFrom(r io.Reader) (n int64, err error) {
@@ -249,7 +249,7 @@ func (u *Utxo) ReadFrom(r io.Reader) (n int64, err error) {
 // WriteTo satisifies the io.WriterTo interface.  A Utxo is written to
 // w in the format:
 //
-//  [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (8 bytes), BlockHash (32 bytes)]
+//  [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (4 bytes), BlockHash (32 bytes)]
 //
 // Each field is written little endian.
 func (u *Utxo) WriteTo(w io.Writer) (n int64, err error) {
@@ -452,7 +452,7 @@ func (txs *TxStore) WriteTo(w io.Writer) (n int64, err error) {
 //
 // Correct results rely on txs being sorted by block height in
 // increasing order.
-func (txs *TxStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool) {
+func (txs *TxStore) Rollback(height int32, hash *btcwire.ShaHash) (modified bool) {
 	s := ([]interface{})(*txs)
 
 	// endlen specifies the final length of the rolled-back TxStore.
@@ -470,7 +470,7 @@ func (txs *TxStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool
 	}()
 
 	for i := len(s) - 1; i >= 0; i-- {
-		var txheight int64
+		var txheight int32
 		var txhash *btcwire.ShaHash
 		switch s[i].(type) {
 		case *RecvTx:
@@ -498,7 +498,7 @@ func (txs *TxStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool
 // ReadFrom satisifies the io.ReaderFrom interface.  A RecTx is read
 // in from r with the format:
 //
-//  [TxHash (32 bytes), BlockHash (32 bytes), Height (8 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)]
+//  [TxHash (32 bytes), BlockHash (32 bytes), Height (4 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)]
 //
 // Each field is read little endian.
 func (tx *RecvTx) ReadFrom(r io.Reader) (n int64, err error) {
@@ -524,7 +524,7 @@ func (tx *RecvTx) ReadFrom(r io.Reader) (n int64, err error) {
 // WriteTo satisifies the io.WriterTo interface.  A RecvTx is written to
 // w in the format:
 //
-//  [TxHash (32 bytes), BlockHash (32 bytes), Height (8 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)]
+//  [TxHash (32 bytes), BlockHash (32 bytes), Height (4 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)]
 //
 // Each field is written little endian.
 func (tx *RecvTx) WriteTo(w io.Writer) (n int64, err error) {
@@ -550,7 +550,7 @@ func (tx *RecvTx) WriteTo(w io.Writer) (n int64, err error) {
 // ReadFrom satisifies the io.WriterTo interface.  A SendTx is read
 // from r with the format:
 //
-//  [TxHash (32 bytes), Height (8 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...]
+//  [TxHash (32 bytes), Height (4 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...]
 //
 // Each field is read little endian.
 func (tx *SendTx) ReadFrom(r io.Reader) (n int64, err error) {
@@ -599,7 +599,7 @@ func (tx *SendTx) ReadFrom(r io.Reader) (n int64, err error) {
 // WriteTo satisifies the io.WriterTo interface.  A SendTx is written to
 // w in the format:
 //
-//  [TxHash (32 bytes), Height (8 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...]
+//  [TxHash (32 bytes), Height (4 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...]
 //
 // Each field is written little endian.
 func (tx *SendTx) WriteTo(w io.Writer) (n int64, err error) {

tx/tx_test.go

@@ -144,7 +144,7 @@ func TestUtxoStoreWriteRead(t *testing.T) {
 		utxo.Out.Index = uint32(i + 2)
 		utxo.Subscript = []byte{}
 		utxo.Amt = uint64(i + 3)
-		utxo.Height = int64(i + 4)
+		utxo.Height = int32(i + 4)
 		*store1 = append(*store1, utxo)
 	}
 

wallet/wallet.go

@@ -34,7 +34,6 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"hash"
 	"io"
-	"math"
 	"math/big"
 	"sync"
 	"time"
@@ -336,6 +335,12 @@ type Wallet struct {
 	highestUsed  int64
 	kdfParams    kdfParameters
 	keyGenerator btcAddress
+
+	// These are non-standard and fit in the extra 1024 bytes between the
+	// root address and the appended entries.
+	syncedBlockHeight int32
+	syncedBlockHash   btcwire.ShaHash
+
 	addrMap        map[[ripemd160.Size]byte]*btcAddress
 	addrCommentMap map[[ripemd160.Size]byte]*[]byte
 	txCommentMap   map[[sha256.Size]byte]*[]byte
@@ -349,11 +354,22 @@ type Wallet struct {
 	lastChainIdx int64
 }
 
+// UnusedWalletBytes specifies the number of actually unused bytes
+// between the root address and the appended entries in a serialized
+// wallet.  Armory's wallet file format provides 1024 unused bytes
+// in this space.  btcwallet requires saving a few additional details
+// with the wallet file, so the binary sizes of those are subtracted
+// from 1024.  Currently, these are:
+//
+//  - last synced block height (int32, 4 bytes)
+//  - last synced block hash (btcwire.ShaHash, btcwire.HashSize bytes)
+const UnusedWalletBytes = 1024 - 4 - btcwire.HashSize
+
 // NewWallet creates and initializes a new Wallet.  name's and
 // desc's binary representation must not exceed 32 and 256 bytes,
 // respectively.  All address private keys are encrypted with passphrase.
 // The wallet is returned unlocked.
-func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*Wallet, error) {
+func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet, createdAt *BlockStamp) (*Wallet, error) {
 	if binary.Size(name) > 32 {
 		return nil, errors.New("name exceeds 32 byte maximum size")
 	}
@@ -366,7 +382,7 @@ func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*W
 	rootkey, chaincode := make([]byte, 32), make([]byte, 32)
 	rand.Read(rootkey)
 	rand.Read(chaincode)
-	root, err := newRootBtcAddress(rootkey, nil, chaincode)
+	root, err := newRootBtcAddress(rootkey, nil, chaincode, createdAt)
 	if err != nil {
 		return nil, err
 	}
@@ -391,6 +407,8 @@ func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*W
 		highestUsed:       -1,
 		kdfParams:         *kdfp,
 		keyGenerator:      *root,
+		syncedBlockHeight: createdAt.Height,
+		syncedBlockHash:   createdAt.Hash,
 		addrMap:           make(map[[ripemd160.Size]byte]*btcAddress),
 		addrCommentMap:    make(map[[ripemd160.Size]byte]*[]byte),
 		txCommentMap:      make(map[[sha256.Size]byte]*[]byte),
@@ -410,7 +428,7 @@ func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*W
 		if err != nil {
 			return nil, err
 		}
-		newaddr, err := newBtcAddress(privkey, nil)
+		newaddr, err := newBtcAddress(privkey, nil, createdAt)
 		if err != nil {
 			return nil, err
 		}
@@ -464,7 +482,9 @@ func (w *Wallet) ReadFrom(r io.Reader) (n int64, err error) {
 		&w.kdfParams,
 		make([]byte, 256),
 		&w.keyGenerator,
-		make([]byte, 1024),
+		&w.syncedBlockHeight,
+		&w.syncedBlockHash,
+		make([]byte, UnusedWalletBytes),
 		&appendedEntries,
 	}
 	for _, data := range datas {
@@ -558,7 +578,9 @@ func (w *Wallet) WriteTo(wtr io.Writer) (n int64, err error) {
 		&w.kdfParams,
 		make([]byte, 256),
 		&w.keyGenerator,
-		make([]byte, 1024),
+		&w.syncedBlockHeight,
+		&w.syncedBlockHash,
+		make([]byte, UnusedWalletBytes),
 		&appendedEntries,
 	}
 	var written int64
@@ -630,9 +652,10 @@ func (w *Wallet) Version() (string, int) {
 	return "", 0
 }
 
-// NextUnusedAddress attempts to get the next chained address.  It
+// NextUnusedAddress attempts to get the next chained address.
-// currently relies on pre-generated addresses and will return an empty
+//
-// string if the address pool has run out. TODO(jrick)
+// TODO(jrick): this currently relies on pre-generated addresses
+// and will return an empty string if the address pool has run out.
 func (w *Wallet) NextUnusedAddress() (string, error) {
 	_ = w.lastChainIdx
 	w.highestUsed++
@@ -701,6 +724,29 @@ func (w *Wallet) Net() btcwire.BitcoinNet {
 	return w.net
 }
 
+// SetSyncedWith marks the wallet to be in sync with the block
+// described by height and hash.
+func (w *Wallet) SetSyncedWith(bs *BlockStamp) {
+	w.syncedBlockHeight = bs.Height
+	copy(w.syncedBlockHash[:], bs.Hash[:])
+}
+
+// SyncedWith returns the height and hash of the block the wallet is
+// currently marked to be in sync with.
+func (w *Wallet) SyncedWith() *BlockStamp {
+	return &BlockStamp{
+		Height: w.syncedBlockHeight,
+		Hash:   w.syncedBlockHash,
+	}
+}
+
+// CreatedAt returns the height of the blockchain at the time of wallet
+// creation.  This is needed when performaing a full rescan to prevent
+// unnecessary rescanning before wallet addresses first appeared.
+func (w *Wallet) CreatedAt() int32 {
+	return w.keyGenerator.firstBlock
+}
+
 func (w *Wallet) addr160ForIdx(idx int64) (*[ripemd160.Size]byte, error) {
 	if idx > w.lastChainIdx {
 		return nil, errors.New("chain index out of range")
@@ -708,21 +754,27 @@ func (w *Wallet) addr160ForIdx(idx int64) (*[ripemd160.Size]byte, error) {
 	return w.chainIdxMap[idx], nil
 }
 
+// AddressInfo holds information regarding an address needed to manage
+// a complete wallet.
+type AddressInfo struct {
+	Address    string
+	FirstBlock int32
+}
+
 // GetActiveAddresses returns all wallet addresses that have been
 // requested to be generated.  These do not include pre-generated
 // addresses.
-func (w *Wallet) GetActiveAddresses() []string {
+func (w *Wallet) GetActiveAddresses() []*AddressInfo {
-	addrs := []string{}
+	addrs := make([]*AddressInfo, 0, w.highestUsed+1)
 	for i := int64(-1); i <= w.highestUsed; i++ {
 		addr160, err := w.addr160ForIdx(i)
 		if err != nil {
 			return addrs
 		}
 		addr := w.addrMap[*addr160]
-		addrstr, err := addr.paymentAddress(w.net)
+		info, err := addr.info(w.Net())
-		// TODO(jrick): propigate error
 		if err == nil {
-			addrs = append(addrs, addrstr)
+			addrs = append(addrs, info)
 		}
 	}
 	return addrs
@@ -817,7 +869,7 @@ type btcAddress struct {
 // newBtcAddress initializes and returns a new address.  privkey must
 // be 32 bytes.  iv must be 16 bytes, or nil (in which case it is
 // randomly generated).
-func newBtcAddress(privkey, iv []byte) (addr *btcAddress, err error) {
+func newBtcAddress(privkey, iv []byte, bs *BlockStamp) (addr *btcAddress, err error) {
 	if len(privkey) != 32 {
 		return nil, errors.New("private key is not 32 bytes")
 	}
@@ -834,8 +886,8 @@ func newBtcAddress(privkey, iv []byte) (addr *btcAddress, err error) {
 			hasPrivKey: true,
 			hasPubKey:  true,
 		},
-		firstSeen:  math.MaxInt64,
+		firstSeen:  time.Now().Unix(),
-		firstBlock: math.MaxInt32,
+		firstBlock: bs.Height,
 	}
 	copy(addr.initVector[:], iv)
 	pub := pubkeyFromPrivkey(privkey)
@@ -848,12 +900,12 @@ func newBtcAddress(privkey, iv []byte) (addr *btcAddress, err error) {
 // newRootBtcAddress generates a new address, also setting the
 // chaincode and chain index to represent this address as a root
 // address.
-func newRootBtcAddress(privKey, iv, chaincode []byte) (addr *btcAddress, err error) {
+func newRootBtcAddress(privKey, iv, chaincode []byte, bs *BlockStamp) (addr *btcAddress, err error) {
 	if len(chaincode) != 32 {
 		return nil, errors.New("chaincode is not 32 bytes")
 	}
 
-	addr, err = newBtcAddress(privKey, iv)
+	addr, err = newBtcAddress(privKey, iv, bs)
 	if err != nil {
 		return nil, err
 	}
@@ -1039,6 +1091,20 @@ func (a *btcAddress) paymentAddress(net btcwire.BitcoinNet) (string, error) {
 	return btcutil.EncodeAddress(a.pubKeyHash[:], net)
 }
 
+// info returns information about a btcAddress stored in a AddressInfo
+// struct.
+func (a *btcAddress) info(net btcwire.BitcoinNet) (*AddressInfo, error) {
+	address, err := a.paymentAddress(net)
+	if err != nil {
+		return nil, err
+	}
+
+	return &AddressInfo{
+		Address:    address,
+		FirstBlock: a.firstBlock,
+	}, nil
+}
+
 func walletHash(b []byte) uint32 {
 	sum := btcwire.DoubleSha256(b)
 	return binary.LittleEndian.Uint32(sum)
@@ -1322,3 +1388,11 @@ func (e *deletedEntry) ReadFrom(r io.Reader) (n int64, err error) {
 	}
 	return n + int64(nRead), err
 }
+
+// BlockStamp defines a block (by height and a unique hash) and is
+// used to mark a point in the blockchain that a wallet element is
+// synced to.
+type BlockStamp struct {
+	Height int32
+	Hash   btcwire.ShaHash
+}

wallet/wallet_test.go

@@ -79,7 +79,9 @@ func TestBtcAddressSerializer(t *testing.T) {
 }
 
 func TestWalletCreationSerialization(t *testing.T) {
-	w1, err := NewWallet("banana wallet", "A wallet for testing.", []byte("banana"), btcwire.MainNet)
+	createdAt := &BlockStamp{}
+	w1, err := NewWallet("banana wallet", "A wallet for testing.",
+		[]byte("banana"), btcwire.MainNet, createdAt)
 	if err != nil {
 		t.Error("Error creating new wallet: " + err.Error())
 	}