waddrmgr/migrations: add migration to populate birthday block for existing wallets

In this commit, we add a new migration to the waddrmgr to populate the
birthday block for existing wallets. This will deem useful when
performing rescans for whatever reason, as we'll now be able to start
from this point rather than the genesis block, incurring a longer
rescan.

The migration is not as reliable since we do not store block timestamps,
so we'll need to estimate our height by looking at the genesis timestamp
and assuming a block occurs every 10 minutes. This can be unsafe, and
cause us to actually miss on-chain events, so a sanity check will be
added before the wallet attempts to sync itself in a later commit.

waddrmgr/migrations.go

@@ -2,7 +2,9 @@ package waddrmgr
 
 import (
 	"fmt"
+	"time"
 
+	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcwallet/walletdb"
 	"github.com/btcsuite/btcwallet/walletdb/migration"
 )
@@ -20,6 +22,10 @@ var versions = []migration.Version{
 		Number:    5,
 		Migration: upgradeToVersion5,
 	},
+	{
+		Number:    6,
+		Migration: populateBirthdayBlock,
+	},
 }
 
 // getLatestVersion returns the version number of the latest database version.
@@ -254,3 +260,93 @@ func migrateRecursively(src, dst walletdb.ReadWriteBucket,
 	}
 	return nil
 }
+
+// populateBirthdayBlock is a migration that attempts to populate the birthday
+// block of the wallet. This is needed so that in the event that we need to
+// perform a rescan of the wallet, we can do so starting from this block, rather
+// than from the genesis block.
+//
+// NOTE: This migration cannot guarantee the correctness of the birthday block
+// being set as we do not store block timestamps, so a sanity check must be done
+// upon starting the wallet to ensure we do not potentially miss any relevant
+// events when rescanning.
+func populateBirthdayBlock(ns walletdb.ReadWriteBucket) error {
+	// We'll need to jump through some hoops in order to determine the
+	// corresponding block height for our birthday timestamp. Since we do
+	// not store block timestamps, we'll need to estimate our height by
+	// looking at the genesis timestamp and assuming a block occurs every 10
+	// minutes. This can be unsafe, and cause us to actually miss on-chain
+	// events, so a sanity check is done before the wallet attempts to sync
+	// itself.
+	//
+	// We'll start by fetching our birthday timestamp.
+	birthdayTimestamp, err := fetchBirthday(ns)
+	if err != nil {
+		return fmt.Errorf("unable to fetch birthday timestamp: %v", err)
+	}
+
+	log.Infof("Setting the wallet's birthday block from timestamp=%v",
+		birthdayTimestamp)
+
+	// Now, we'll need to determine the timestamp of the genesis block for
+	// the corresponding chain.
+	genesisHash, err := fetchBlockHash(ns, 0)
+	if err != nil {
+		return fmt.Errorf("unable to fetch genesis block hash: %v", err)
+	}
+
+	var genesisTimestamp time.Time
+	switch *genesisHash {
+	case *chaincfg.MainNetParams.GenesisHash:
+		genesisTimestamp =
+			chaincfg.MainNetParams.GenesisBlock.Header.Timestamp
+
+	case *chaincfg.TestNet3Params.GenesisHash:
+		genesisTimestamp =
+			chaincfg.TestNet3Params.GenesisBlock.Header.Timestamp
+
+	case *chaincfg.RegressionNetParams.GenesisHash:
+		genesisTimestamp =
+			chaincfg.RegressionNetParams.GenesisBlock.Header.Timestamp
+
+	case *chaincfg.SimNetParams.GenesisHash:
+		genesisTimestamp =
+			chaincfg.SimNetParams.GenesisBlock.Header.Timestamp
+
+	default:
+		return fmt.Errorf("unknown genesis hash %v", genesisHash)
+	}
+
+	// With the timestamps retrieved, we can estimate a block height by
+	// taking the difference between them and dividing by the average block
+	// time (10 minutes).
+	birthdayHeight := int32((birthdayTimestamp.Sub(genesisTimestamp).Seconds() / 600))
+
+	// Now that we have the height estimate, we can fetch the corresponding
+	// block and set it as our birthday block.
+	birthdayHash, err := fetchBlockHash(ns, birthdayHeight)
+
+	// To ensure we record a height that is known to us from the chain,
+	// we'll make sure this height estimate can be found. Otherwise, we'll
+	// continue subtracting a day worth of blocks until we can find one.
+	for IsError(err, ErrBlockNotFound) {
+		birthdayHeight -= 144
+		if birthdayHeight < 0 {
+			birthdayHeight = 0
+		}
+		birthdayHash, err = fetchBlockHash(ns, birthdayHeight)
+	}
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Estimated birthday block from timestamp=%v: height=%d, "+
+		"hash=%v", birthdayTimestamp, birthdayHeight, birthdayHash)
+
+	// NOTE: The timestamp of the birthday block isn't set since we do not
+	// store each block's timestamp.
+	return putBirthdayBlock(ns, BlockStamp{
+		Height: birthdayHeight,
+		Hash:   *birthdayHash,
+	})
+}

waddrmgr/migrations_test.go

@@ -0,0 +1,217 @@
+package waddrmgr
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcwallet/walletdb"
+)
+
+// applyMigration is a helper function that allows us to assert the state of the
+// top-level bucket before and after a migration. This can be used to ensure
+// the correctness of migrations.
+func applyMigration(t *testing.T, beforeMigration, afterMigration,
+	migration func(walletdb.ReadWriteBucket) error, shouldFail bool) {
+
+	t.Helper()
+
+	// We'll start by setting up our address manager backed by a database.
+	teardown, db, _ := setupManager(t)
+	defer teardown()
+
+	// First, we'll run the beforeMigration closure, which contains the
+	// database modifications/assertions needed before proceeding with the
+	// migration.
+	err := walletdb.Update(db, func(tx walletdb.ReadWriteTx) error {
+		ns := tx.ReadWriteBucket(waddrmgrNamespaceKey)
+		if ns == nil {
+			return errors.New("top-level namespace does not exist")
+		}
+		return beforeMigration(ns)
+	})
+	if err != nil {
+		t.Fatalf("unable to run beforeMigration func: %v", err)
+	}
+
+	// Then, we'll run the migration itself and fail if it does not match
+	// its expected result.
+	err = walletdb.Update(db, func(tx walletdb.ReadWriteTx) error {
+		ns := tx.ReadWriteBucket(waddrmgrNamespaceKey)
+		if ns == nil {
+			return errors.New("top-level namespace does not exist")
+		}
+		return migration(ns)
+	})
+	if err != nil && !shouldFail {
+		t.Fatalf("unable to perform migration: %v", err)
+	} else if err == nil && shouldFail {
+		t.Fatal("expected migration to fail, but did not")
+	}
+
+	// Finally, we'll run the afterMigration closure, which contains the
+	// assertions needed in order to guarantee than the migration was
+	// successful.
+	err = walletdb.Update(db, func(tx walletdb.ReadWriteTx) error {
+		ns := tx.ReadWriteBucket(waddrmgrNamespaceKey)
+		if ns == nil {
+			return errors.New("top-level namespace does not exist")
+		}
+		return afterMigration(ns)
+	})
+	if err != nil {
+		t.Fatalf("unable to run afterMigration func: %v", err)
+	}
+}
+
+// TestMigrationPupulateBirthdayBlock ensures that the migration to populate the
+// wallet's birthday block works as intended.
+func TestMigrationPopulateBirthdayBlock(t *testing.T) {
+	t.Parallel()
+
+	var expectedHeight int32
+	beforeMigration := func(ns walletdb.ReadWriteBucket) error {
+		// To test this migration, we'll start by writing to disk 10
+		// random blocks.
+		block := &BlockStamp{}
+		for i := int32(1); i <= 10; i++ {
+			block.Height = i
+			blockHash := bytes.Repeat([]byte(string(i)), 32)
+			copy(block.Hash[:], blockHash)
+			if err := putSyncedTo(ns, block); err != nil {
+				return err
+			}
+		}
+
+		// With the blocks inserted, we'll assume that the birthday
+		// block corresponds to the 7th block (out of 11) in the chain.
+		// To do this, we'll need to set our birthday timestamp to the
+		// estimated timestamp of a block that's 6 blocks after genesis.
+		genesisTimestamp := chaincfg.MainNetParams.GenesisBlock.Header.Timestamp
+		delta := time.Hour
+		expectedHeight = int32(delta.Seconds() / 600)
+		birthday := genesisTimestamp.Add(delta)
+		if err := putBirthday(ns, birthday); err != nil {
+			return err
+		}
+
+		// Finally, since the migration has not yet started, we should
+		// not be able to find the birthday block within the database.
+		_, err := fetchBirthdayBlock(ns)
+		if !IsError(err, ErrBirthdayBlockNotSet) {
+			return fmt.Errorf("expected ErrBirthdayBlockNotSet, "+
+				"got %v", err)
+		}
+
+		return nil
+	}
+
+	// After the migration has completed, we should see that the birthday
+	// block now exists and is set to the correct expected height.
+	afterMigration := func(ns walletdb.ReadWriteBucket) error {
+		birthdayBlock, err := fetchBirthdayBlock(ns)
+		if err != nil {
+			return err
+		}
+
+		if birthdayBlock.Height != expectedHeight {
+			return fmt.Errorf("expected birthday block with "+
+				"height %d, got %d", expectedHeight,
+				birthdayBlock.Height)
+		}
+
+		return nil
+	}
+
+	// We can now apply the migration and expect it not to fail.
+	applyMigration(
+		t, beforeMigration, afterMigration, populateBirthdayBlock,
+		false,
+	)
+}
+
+// TestMigrationPopulateBirthdayBlockEstimateTooFar ensures that the migration
+// can properly detect a height estimate which the chain from our point of view
+// has not yet reached.
+func TestMigrationPopulateBirthdayBlockEstimateTooFar(t *testing.T) {
+	t.Parallel()
+
+	const numBlocks = 1000
+	chainParams := chaincfg.MainNetParams
+
+	var expectedHeight int32
+	beforeMigration := func(ns walletdb.ReadWriteBucket) error {
+		// To test this migration, we'll start by writing to disk 999
+		// random blocks to simulate a synced chain with height 1000.
+		block := &BlockStamp{}
+		for i := int32(1); i < numBlocks; i++ {
+			block.Height = i
+			blockHash := bytes.Repeat([]byte(string(i)), 32)
+			copy(block.Hash[:], blockHash)
+			if err := putSyncedTo(ns, block); err != nil {
+				return err
+			}
+		}
+
+		// With the blocks inserted, we'll assume that the birthday
+		// block corresponds to the 900th block in the chain. To do
+		// this, we'd need to set our birthday timestamp to the
+		// estimated timestamp of a block that's 899 blocks after
+		// genesis. However, this will not work if the average block
+		// time is not 10 mins, which can throw off the height estimate
+		// with a height longer than the chain in the event of test
+		// networks (testnet, regtest, etc. and not fully synced
+		// wallets). Instead the migration should be able to handle this
+		// by subtracting a days worth of blocks until finding a block
+		// that it is aware of.
+		//
+		// We'll have the migration assume that our birthday is at block
+		// 1001 in the chain. Since this block doesn't exist from the
+		// database's point of view, a days worth of blocks will be
+		// subtracted from the estimate, which should give us a valid
+		// block height.
+		genesisTimestamp := chainParams.GenesisBlock.Header.Timestamp
+		delta := numBlocks * 10 * time.Minute
+		expectedHeight = numBlocks - 144
+
+		birthday := genesisTimestamp.Add(delta)
+		if err := putBirthday(ns, birthday); err != nil {
+			return err
+		}
+
+		// Finally, since the migration has not yet started, we should
+		// not be able to find the birthday block within the database.
+		_, err := fetchBirthdayBlock(ns)
+		if !IsError(err, ErrBirthdayBlockNotSet) {
+			return fmt.Errorf("expected ErrBirthdayBlockNotSet, "+
+				"got %v", err)
+		}
+
+		return nil
+	}
+
+	// After the migration has completed, we should see that the birthday
+	// block now exists and is set to the correct expected height.
+	afterMigration := func(ns walletdb.ReadWriteBucket) error {
+		birthdayBlock, err := fetchBirthdayBlock(ns)
+		if err != nil {
+			return err
+		}
+
+		if birthdayBlock.Height != expectedHeight {
+			return fmt.Errorf("expected birthday block height %d, "+
+				"got %d", expectedHeight, birthdayBlock.Height)
+		}
+
+		return nil
+	}
+
+	// We can now apply the migration and expect it not to fail.
+	applyMigration(
+		t, beforeMigration, afterMigration, populateBirthdayBlock,
+		false,
+	)
+}