Rescan and track sync status of imported addresses.

The private key import codepath (called when handling the
importprivkey RPC method) was not triggering rescans for the imported
address.  This change begins a new rescan for each import and adds
additional logic to the wallet file to keep track of unsynced imported
addresses.  After a rescan on an imported address completes, the
address is marked as in sync with the rest of wallet and future
handshake rescans will start from the last seen block, rather than the
import height of the unsynced address.

While here, improve the logging for not just import rescans, but
rescanning on btcd connect (part of the handshake) as well.

Fixes #74.

account.go

@@ -352,7 +352,9 @@ func (a *Account) DumpWIFPrivateKey(addr btcutil.Address) (string, error) {
 
 // ImportPrivateKey imports a private key to the account's wallet and
 // writes the new wallet to disk.
-func (a *Account) ImportPrivateKey(pk []byte, compressed bool, bs *wallet.BlockStamp) (string, error) {
+func (a *Account) ImportPrivateKey(pk []byte, compressed bool,
+	bs *wallet.BlockStamp, rescan bool) (string, error) {
+
 	// Attempt to import private key into wallet.
 	addr, err := a.Wallet.ImportPrivateKey(pk, compressed, bs)
 	if err != nil {
@@ -366,6 +368,46 @@ func (a *Account) ImportPrivateKey(pk []byte, compressed bool, bs *wallet.BlockS
 		return "", fmt.Errorf("cannot write account: %v", err)
 	}
 
+	// Rescan blockchain for transactions with txout scripts paying to the
+	// imported address.
+	//
+	// TODO(jrick): As btcd only allows a single rescan per websocket client
+	// to run at any given time, a separate goroutine should run for
+	// exclusively handling rescan events.
+	if rescan {
+		go func(addr btcutil.Address, aname string) {
+			addrStr := addr.EncodeAddress()
+			log.Infof("Beginning rescan (height %d) for address %s",
+				bs.Height, addrStr)
+
+			rescanAddrs := map[string]struct{}{
+				addrStr: struct{}{},
+			}
+			jsonErr := Rescan(CurrentServerConn(), bs.Height,
+				rescanAddrs)
+			if jsonErr != nil {
+				log.Errorf("Rescan for imported address %s failed: %v",
+					addrStr, jsonErr.Message)
+				return
+			}
+
+			AcctMgr.Grab()
+			defer AcctMgr.Release()
+			a, err := AcctMgr.Account(aname)
+			if err != nil {
+				log.Errorf("Account for imported address %s missing: %v",
+					addrStr, err)
+				return
+			}
+			if err := a.MarkAddressSynced(addr); err != nil {
+				log.Errorf("Unable to mark rescanned address as synced: %v", err)
+				return
+			}
+			AcctMgr.ds.FlushAccount(a)
+			log.Infof("Finished rescan for imported address %s", addrStr)
+		}(addr, a.name)
+	}
+
 	// Associate the imported address with this account.
 	MarkAddressForAccount(addrStr, a.Name())
 
@@ -454,28 +496,26 @@ func (a *Account) Track() {
 // main chain.
 func (a *Account) RescanActiveAddresses() {
 	// Determine the block to begin the rescan from.
-	beginBlock := int32(0)
+	height := int32(0)
 	if a.fullRescan {
 		// Need to perform a complete rescan since the wallet creation
 		// block.
-		beginBlock = a.EarliestBlockHeight()
+		height = a.EarliestBlockHeight()
-		log.Debugf("Rescanning account '%v' for new transactions since block height %v",
-			a.name, beginBlock)
 	} else {
 		// The last synced block height should be used the starting
 		// point for block rescanning.  Grab the block stamp here.
-		bs := a.SyncedWith()
+		height = a.SyncHeight()
-
-		log.Debugf("Rescanning account '%v' for new transactions after block height %v hash %v",
-			a.name, bs.Height, bs.Hash)
-
-		// If we're synced with block x, must scan the blocks x+1 to best block.
-		beginBlock = bs.Height + 1
 	}
 
+	log.Info("Beginning rescan (height %d) for account '%v'",
+		height, a.name)
+
 	// Rescan active addresses starting at the determined block height.
-	Rescan(CurrentServerConn(), beginBlock, a.ActivePaymentAddresses())
+	Rescan(CurrentServerConn(), height, a.ActivePaymentAddresses())
+	a.MarkAllSynced()
 	AcctMgr.ds.FlushAccount(a)
+
+	log.Info("Finished rescan for account '%v'", a.name)
 }
 
 func (a *Account) ResendUnminedTxs() {

ntfns.go

@@ -72,7 +72,7 @@ func NtfnRecvTx(n btcjson.Cmd) error {
 
 	rawTx, err := hex.DecodeString(rtx.HexTx)
 	if err != nil {
-		return fmt.Errorf("%v handler: bad hexstring: err", n.Method(), err)
+		return fmt.Errorf("%v handler: bad hexstring: %v", n.Method(), err)
 	}
 	tx_, err := btcutil.NewTxFromBytes(rawTx)
 	if err != nil {
@@ -248,7 +248,7 @@ func NtfnRedeemingTx(n btcjson.Cmd) error {
 
 	rawTx, err := hex.DecodeString(cn.HexTx)
 	if err != nil {
-		return fmt.Errorf("%v handler: bad hexstring: err", n.Method(), err)
+		return fmt.Errorf("%v handler: bad hexstring: %v", n.Method(), err)
 	}
 	tx_, err := btcutil.NewTxFromBytes(rawTx)
 	if err != nil {

rpcserver.go

@@ -643,7 +643,7 @@ func ImportPrivKey(icmd btcjson.Cmd) (interface{}, *btcjson.Error) {
 
 	// Import the private key, handling any errors.
 	bs := &wallet.BlockStamp{}
-	switch _, err := a.ImportPrivateKey(pk, compressed, bs); err {
+	switch _, err := a.ImportPrivateKey(pk, compressed, bs, cmd.Rescan); err {
 	case nil:
 		// If the import was successful, reply with nil.
 		return nil, nil

wallet/wallet.go

@@ -871,7 +871,7 @@ func (w *Wallet) Lock() (err error) {
 
 	// Remove clear text private keys from all address entries.
 	for _, addr := range w.addrMap {
-		if baddr, ok :=  addr.(*btcAddress); ok {
+		if baddr, ok := addr.(*btcAddress); ok {
 			_ = baddr.lock()
 		}
 	}
@@ -1252,8 +1252,31 @@ func (w *Wallet) Net() btcwire.BitcoinNet {
 	return w.net
 }
 
-// SetSyncedWith marks the wallet to be in sync with the block
+// MarkAddressSynced marks an unsynced (likely imported) address as
-// described by height and hash.
+// being fully in sync with the rest of wallet.
+func (w *Wallet) MarkAddressSynced(a btcutil.Address) error {
+	wa, ok := w.addrMap[getAddressKey(a)]
+	if !ok {
+		return ErrAddressNotFound
+	}
+	wa.markSynced()
+	return nil
+}
+
+// MarkAllSynced marks all unsynced (likely imported) wallet addresses
+// as being fully in sync with marked recently-seen blocks (marked
+// using SetSyncedWith).
+func (w *Wallet) MarkAllSynced() {
+	for _, wa := range w.addrMap {
+		wa.markSynced()
+	}
+}
+
+// SetSyncedWith marks already synced addresses in the wallet to be in
+// sync with the recently-seen block described by the blockstamp.
+// Unsynced addresses are unaffected by this method and must be marked
+// as in sync with MarkAddressSynced or MarkAllSynced to be considered
+// in sync with bs.
 func (w *Wallet) SetSyncedWith(bs *BlockStamp) {
 	// Check if we're trying to rollback the last seen history.
 	// If so, and this bs is already saved, remove anything
@@ -1289,21 +1312,29 @@ func (w *Wallet) SetSyncedWith(bs *BlockStamp) {
 	}
 }
 
-// SyncedWith returns the height and hash of the block the wallet is
+// SyncHeight returns the sync height of a wallet, or the earliest
-// currently marked to be in sync with.
+// block height of any unsynced imported address if there are any
-func (w *Wallet) SyncedWith() *BlockStamp {
+// addresses marked as unsynced, whichever is smaller.  This is the
-	nHashes := len(w.recent.hashes)
+// height that rescans on an entire wallet should begin at to fully
-	if nHashes == 0 || w.recent.lastHeight == -1 {
+// sync all wallet addresses.
-		return &BlockStamp{
+func (w *Wallet) SyncHeight() (height int32) {
-			Height: -1,
+	if len(w.recent.hashes) == 0 {
+		return 0
+	}
+	height = w.recent.lastHeight
+
+	for _, a := range w.addrMap {
+		if a.unsynced() && a.firstBlockHeight() < height {
+			height = a.firstBlockHeight()
+
+			// Can't go lower than 0.
+			if height == 0 {
+				break
+			}
 		}
 	}
 
-	lastSha := w.recent.hashes[nHashes-1]
+	return height
-	return &BlockStamp{
-		Height: w.recent.lastHeight,
-		Hash:   *lastSha,
-	}
 }
 
 // NewIterateRecentBlocks returns an iterator for recently-seen blocks.
@@ -1375,6 +1406,12 @@ func (w *Wallet) ImportPrivateKey(privkey []byte, compressed bool, bs *BlockStam
 	}
 	btcaddr.chainIndex = importedKeyChainIdx
 
+	// Mark as unsynced if import height is below currently-synced
+	// height.
+	if len(w.recent.hashes) != 0 && bs.Height < w.recent.lastHeight {
+		btcaddr.flags.unsynced = true
+	}
+
 	// Encrypt imported address with the derived AES key.
 	if err = btcaddr.encrypt(w.secret); err != nil {
 		return nil, err
@@ -1408,6 +1445,12 @@ func (w *Wallet) ImportScript(script []byte, bs *BlockStamp) (btcutil.Address, e
 		return nil, err
 	}
 
+	// Mark as unsynced if import height is below currently-synced
+	// height.
+	if len(w.recent.hashes) != 0 && bs.Height < w.recent.lastHeight {
+		scriptaddr.flags.unsynced = true
+	}
+
 	// Add address to wallet's bookkeeping structures.  Adding to
 	// the map will result in the imported address being serialized
 	// on the next WriteTo call.
@@ -1675,6 +1718,7 @@ type addrFlags struct {
 	createPrivKeyNextUnlock bool
 	compressed              bool
 	change                  bool
+	unsynced                bool
 }
 
 func (af *addrFlags) ReadFrom(r io.Reader) (int64, error) {
@@ -1690,6 +1734,7 @@ func (af *addrFlags) ReadFrom(r io.Reader) (int64, error) {
 	af.createPrivKeyNextUnlock = b[0]&(1<<3) != 0
 	af.compressed = b[0]&(1<<4) != 0
 	af.change = b[0]&(1<<5) != 0
+	af.unsynced = b[0]&(1<<6) != 0
 
 	// Currently (at least until watching-only wallets are implemented)
 	// btcwallet shall refuse to open any unencrypted addresses.  This
@@ -1727,6 +1772,9 @@ func (af *addrFlags) WriteTo(w io.Writer) (int64, error) {
 	if af.change {
 		b[0] |= 1 << 5
 	}
+	if af.unsynced {
+		b[0] |= 1 << 6
+	}
 
 	n, err := w.Write(b[:])
 	return int64(n), err
@@ -1998,6 +2046,8 @@ type walletAddress interface {
 	watchingCopy() walletAddress
 	firstBlockHeight() int32
 	imported() bool
+	unsynced() bool
+	markSynced()
 }
 
 type btcAddress struct {
@@ -2098,6 +2148,7 @@ func newBtcAddress(privkey, iv []byte, bs *BlockStamp, compressed bool) (addr *b
 			createPrivKeyNextUnlock: false,
 			compressed:              compressed,
 			change:                  false,
+			unsynced:                false,
 		},
 		firstSeen:  time.Now().Unix(),
 		firstBlock: bs.Height,
@@ -2143,6 +2194,7 @@ func newBtcAddressWithoutPrivkey(pubkey, iv []byte, bs *BlockStamp) (addr *btcAd
 			createPrivKeyNextUnlock: true,
 			compressed:              compressed,
 			change:                  false,
+			unsynced:                false,
 		},
 		firstSeen:  time.Now().Unix(),
 		firstBlock: bs.Height,
@@ -2461,6 +2513,7 @@ func (a *btcAddress) watchingCopy() walletAddress {
 			createPrivKeyNextUnlock: false,
 			compressed:              a.flags.compressed,
 			change:                  a.flags.change,
+			unsynced:                a.flags.unsynced,
 		},
 		chaincode:  a.chaincode,
 		chainIndex: a.chainIndex,
@@ -2481,6 +2534,14 @@ func (a *btcAddress) imported() bool {
 	return a.chainIndex == importedKeyChainIdx
 }
 
+func (a *btcAddress) unsynced() bool {
+	return a.flags.unsynced
+}
+
+func (a *btcAddress) markSynced() {
+	a.flags.unsynced = false
+}
+
 // note that there is no encrypted bit here since if we had a script encrypted
 // and then used it on the blockchain this provides a simple known plaintext in
 // the wallet file. It was determined that the script in a p2sh transaction is
@@ -2489,6 +2550,7 @@ func (a *btcAddress) imported() bool {
 type scriptFlags struct {
 	hasScript bool
 	change    bool
+	unsynced  bool
 }
 
 // ReadFrom implements the io.ReaderFrom interface by reading from r into sf.
@@ -2503,6 +2565,7 @@ func (sf *scriptFlags) ReadFrom(r io.Reader) (int64, error) {
 	// the same bit as hasPubKey and the change bit is the same for both.
 	sf.hasScript = b[0]&(1<<1) != 0
 	sf.change = b[0]&(1<<5) != 0
+	sf.unsynced = b[0]&(1<<6) != 0
 
 	return int64(n), nil
 }
@@ -2516,6 +2579,9 @@ func (sf *scriptFlags) WriteTo(w io.Writer) (int64, error) {
 	if sf.change {
 		b[0] |= 1 << 5
 	}
+	if sf.unsynced {
+		b[0] |= 1 << 6
+	}
 
 	n, err := w.Write(b[:])
 	return int64(n), err
@@ -2786,6 +2852,14 @@ func (a *scriptAddress) imported() bool {
 	return true
 }
 
+func (a *scriptAddress) unsynced() bool {
+	return a.flags.unsynced
+}
+
+func (a *scriptAddress) markSynced() {
+	a.flags.unsynced = false
+}
+
 func walletHash(b []byte) uint32 {
 	sum := btcwire.DoubleSha256(b)
 	return binary.LittleEndian.Uint32(sum)

wallet/wallet_test.go

@@ -707,7 +707,8 @@ func TestWatchingWalletExport(t *testing.T) {
 
 func TestImportPrivateKey(t *testing.T) {
 	const keypoolSize = 10
-	createdAt := &BlockStamp{}
+	createHeight := int32(100)
+	createdAt := &BlockStamp{Height: createHeight}
 	w, err := NewWallet("banana wallet", "A wallet for testing.",
 		[]byte("banana"), btcwire.MainNet, createdAt, keypoolSize)
 	if err != nil {
@@ -726,9 +727,21 @@ func TestImportPrivateKey(t *testing.T) {
 		return
 	}
 
+	// verify that the entire wallet's sync height matches the
+	// expected createHeight.
+	if h := w.EarliestBlockHeight(); h != createHeight {
+		t.Error("Initial earliest height %v does not match expected %v.", h, createHeight)
+		return
+	}
+	if h := w.SyncHeight(); h != createHeight {
+		t.Error("Initial sync height %v does not match expected %v.", h, createHeight)
+		return
+	}
+
 	// import priv key
-	stamp := &BlockStamp{}
+	importHeight := int32(50)
-	address, err := w.ImportPrivateKey(pk.D.Bytes(), false, stamp)
+	importedAt := &BlockStamp{Height: importHeight}
+	address, err := w.ImportPrivateKey(pk.D.Bytes(), false, importedAt)
 	if err != nil {
 		t.Error("importing private key: " + err.Error())
 		return
@@ -745,6 +758,17 @@ func TestImportPrivateKey(t *testing.T) {
 		return
 	}
 
+	// verify that the earliest block and sync heights now match the
+	// (smaller) import height.
+	if h := w.EarliestBlockHeight(); h != importHeight {
+		t.Errorf("After import earliest height %v does not match expected %v.", h, importHeight)
+		return
+	}
+	if h := w.SyncHeight(); h != importHeight {
+		t.Errorf("After import sync height %v does not match expected %v.", h, importHeight)
+		return
+	}
+
 	// serialise and deseralise and check still there.
 
 	// Test (de)serialization of wallet.
@@ -761,6 +785,34 @@ func TestImportPrivateKey(t *testing.T) {
 		return
 	}
 
+	// Verify that the earliest and sync height match expected after the reserialization.
+	if h := w2.EarliestBlockHeight(); h != importHeight {
+		t.Errorf("After reserialization earliest height %v does not match expected %v.", h, importHeight)
+		return
+	}
+	if h := w2.SyncHeight(); h != importHeight {
+		t.Errorf("After reserialization sync height %v does not match expected %v.", h, importHeight)
+		return
+	}
+
+	if err := w2.MarkAddressSynced(address); err != nil {
+		t.Errorf("Cannot mark address synced: %v", err)
+		return
+	}
+
+	// Mark imported address as synced with the recently-seen blocks, and verify
+	// that the sync height now equals the most recent block (the one at wallet
+	// creation).
+	w2.MarkAddressSynced(address)
+	if h := w2.EarliestBlockHeight(); h != importHeight {
+		t.Errorf("After address sync, earliest height %v does not match expected %v.", h, importHeight)
+		return
+	}
+	if h := w2.SyncHeight(); h != createHeight {
+		t.Errorf("After address sync, sync height %v does not match expected %v.", h, createHeight)
+		return
+	}
+
 	if err = w2.Unlock([]byte("banana")); err != nil {
 		t.Errorf("Can't unlock deserialised wallet: %v", err)
 		return
@@ -781,7 +833,8 @@ func TestImportPrivateKey(t *testing.T) {
 
 func TestImportScript(t *testing.T) {
 	const keypoolSize = 10
-	createdAt := &BlockStamp{}
+	createHeight := int32(100)
+	createdAt := &BlockStamp{Height: createHeight}
 	w, err := NewWallet("banana wallet", "A wallet for testing.",
 		[]byte("banana"), btcwire.MainNet, createdAt, keypoolSize)
 	if err != nil {
@@ -794,9 +847,21 @@ func TestImportScript(t *testing.T) {
 		return
 	}
 
+	// verify that the entire wallet's sync height matches the
+	// expected createHeight.
+	if h := w.EarliestBlockHeight(); h != createHeight {
+		t.Error("Initial earliest height %v does not match expected %v.", h, createHeight)
+		return
+	}
+	if h := w.SyncHeight(); h != createHeight {
+		t.Error("Initial sync height %v does not match expected %v.", h, createHeight)
+		return
+	}
+
 	script := []byte{btcscript.OP_TRUE, btcscript.OP_DUP,
 		btcscript.OP_DROP}
-	stamp := &BlockStamp{}
+	importHeight := int32(50)
+	stamp := &BlockStamp{Height: importHeight}
 	address, err := w.ImportScript(script, stamp)
 	if err != nil {
 		t.Error("error importing script: " + err.Error())
@@ -845,7 +910,7 @@ func TestImportScript(t *testing.T) {
 		return
 	}
 
-	if sinfo.FirstBlock() != 0 {
+	if sinfo.FirstBlock() != importHeight {
 		t.Error("funny first block")
 		return
 	}
@@ -865,6 +930,17 @@ func TestImportScript(t *testing.T) {
 		return
 	}
 
+	// verify that the earliest block and sync heights now match the
+	// (smaller) import height.
+	if h := w.EarliestBlockHeight(); h != importHeight {
+		t.Errorf("After import earliest height %v does not match expected %v.", h, importHeight)
+		return
+	}
+	if h := w.SyncHeight(); h != importHeight {
+		t.Errorf("After import sync height %v does not match expected %v.", h, importHeight)
+		return
+	}
+
 	// serialise and deseralise and check still there.
 
 	// Test (de)serialization of wallet.
@@ -881,6 +957,34 @@ func TestImportScript(t *testing.T) {
 		return
 	}
 
+	// Verify that the earliest and sync height match expected after the reserialization.
+	if h := w2.EarliestBlockHeight(); h != importHeight {
+		t.Errorf("After reserialization earliest height %v does not match expected %v.", h, importHeight)
+		return
+	}
+	if h := w2.SyncHeight(); h != importHeight {
+		t.Errorf("After reserialization sync height %v does not match expected %v.", h, importHeight)
+		return
+	}
+
+	if err := w2.MarkAddressSynced(address); err != nil {
+		t.Errorf("Cannot mark address synced: %v", err)
+		return
+	}
+
+	// Mark imported address as synced with the recently-seen blocks, and verify
+	// that the sync height now equals the most recent block (the one at wallet
+	// creation).
+	w2.MarkAddressSynced(address)
+	if h := w2.EarliestBlockHeight(); h != importHeight {
+		t.Errorf("After address sync, earliest height %v does not match expected %v.", h, importHeight)
+		return
+	}
+	if h := w2.SyncHeight(); h != createHeight {
+		t.Errorf("After address sync, sync height %v does not match expected %v.", h, createHeight)
+		return
+	}
+
 	if err = w2.Unlock([]byte("banana")); err != nil {
 		t.Errorf("Can't unlock deserialised wallet: %v", err)
 		return