From 74fb6e56da4d09cef109e3cd499df5bb45f82151 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 5 Sep 2017 18:24:52 -0700 Subject: [PATCH] blockchain: Database migration to populate block index bucket. This creates a migration function that populates the block index bucket using data from the ffldb block index bucket if it does not exist. --- blockchain/blockindex.go | 2 +- blockchain/chainio.go | 45 ++++++++----- blockchain/upgrade.go | 139 +++++++++++++++++++++++++++++++++++++++ blockchain/validate.go | 6 +- 4 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 blockchain/upgrade.go diff --git a/blockchain/blockindex.go b/blockchain/blockindex.go index 642b658e..e9be63b3 100644 --- a/blockchain/blockindex.go +++ b/blockchain/blockindex.go @@ -136,7 +136,7 @@ func newBlockNode(blockHeader *wire.BlockHeader, height int32) *blockNode { // This function is safe for concurrent access. func (node *blockNode) Header() wire.BlockHeader { // No lock is needed because all accessed fields are immutable. - prevHash := zeroHash + prevHash := &zeroHash if node.parent != nil { prevHash = &node.parent.hash } diff --git a/blockchain/chainio.go b/blockchain/chainio.go index 66b552de..51216f56 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1148,17 +1148,39 @@ func (b *BlockChain) createChainState() error { // database. When the db does not yet contain any chain state, both it and the // chain state are initialized to the genesis block. func (b *BlockChain) initChainState() error { - // Attempt to load the chain state from the database. - var isStateInitialized bool + // Determine the state of the chain database. We may need to initialize + // everything from scratch or upgrade certain buckets. + var initialized bool + var hasBlockIndex bool err := b.db.View(func(dbTx database.Tx) error { + initialized = dbTx.Metadata().Get(chainStateKeyName) != nil + hasBlockIndex = dbTx.Metadata().Bucket(blockIndexBucketName) != nil + return nil + }) + if err != nil { + return err + } + + if !initialized { + // At this point the database has not already been initialized, so + // initialize both it and the chain state to the genesis block. + return b.createChainState() + } + + if !hasBlockIndex { + err := migrateBlockIndex(b.db) + if err != nil { + return nil + } + } + + // Attempt to load the chain state from the database. + return b.db.View(func(dbTx database.Tx) error { // Fetch the stored chain state from the database metadata. // When it doesn't exist, it means the database hasn't been // initialized for use with chain yet, so break out now to allow // that to happen under a writable database transaction. serializedData := dbTx.Metadata().Get(chainStateKeyName) - if serializedData == nil { - return nil - } log.Tracef("Serialized chain state: %x", serializedData) state, err := deserializeBestChainState(serializedData) if err != nil { @@ -1253,22 +1275,9 @@ func (b *BlockChain) initChainState() error { numTxns := uint64(len(block.Transactions)) b.stateSnapshot = newBestState(tip, blockSize, blockWeight, numTxns, state.totalTxns, tip.CalcPastMedianTime()) - isStateInitialized = true return nil }) - if err != nil { - return err - } - - // There is nothing more to do if the chain state was initialized. - if isStateInitialized { - return nil - } - - // At this point the database has not already been initialized, so - // initialize both it and the chain state to the genesis block. - return b.createChainState() } // dbFetchHeaderByHash uses an existing database transaction to retrieve the diff --git a/blockchain/upgrade.go b/blockchain/upgrade.go new file mode 100644 index 00000000..60e8a897 --- /dev/null +++ b/blockchain/upgrade.go @@ -0,0 +1,139 @@ +// Copyright (c) 2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "bytes" + "container/list" + "fmt" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/database" + "github.com/btcsuite/btcd/wire" +) + +const ( + // blockHdrOffset defines the offsets into a v1 block index row for the + // block header. + // + // The serialized block index row format is: + // + blockHdrOffset = 12 +) + +// migrateBlockIndex migrates all block entries from the v1 block index bucket +// to the v2 bucket. The v1 bucket stores all block entries keyed by block hash, +// whereas the v2 bucket stores the exact same values, but keyed instead by +// block height + hash. +func migrateBlockIndex(db database.DB) error { + // Hardcoded bucket names so updates to the global values do not affect + // old upgrades. + v1BucketName := []byte("ffldb-blockidx") + v2BucketName := []byte("blockheaderidx") + + err := db.Update(func(dbTx database.Tx) error { + v1BlockIdxBucket := dbTx.Metadata().Bucket(v1BucketName) + if v1BlockIdxBucket == nil { + return fmt.Errorf("Bucket %s does not exist", v1BucketName) + } + + log.Info("Re-indexing block information in the database. This might take a while...") + + v2BlockIdxBucket, err := + dbTx.Metadata().CreateBucketIfNotExists(v2BucketName) + if err != nil { + return err + } + + // Scan the old block index bucket and construct a mapping of each block + // to all child blocks. + childBlocksMap, err := readBlockTree(v1BlockIdxBucket) + if err != nil { + return err + } + + // Use the block graph to calculate the height of each block. + blockHeights := determineBlockHeights(childBlocksMap) + + // Now that we have heights for all blocks, scan the old block index + // bucket and insert all rows into the new one. + return v1BlockIdxBucket.ForEach(func(hashBytes, blockRow []byte) error { + endOffset := blockHdrOffset + blockHdrSize + headerBytes := blockRow[blockHdrOffset:endOffset:endOffset] + + var hash chainhash.Hash + copy(hash[:], hashBytes[0:chainhash.HashSize]) + + height, exists := blockHeights[hash] + if !exists { + return fmt.Errorf("Unable to calculate chain height for "+ + "stored block %s", hash) + } + + key := blockIndexKey(&hash, height) + return v2BlockIdxBucket.Put(key, headerBytes) + }) + }) + if err != nil { + return err + } + + log.Infof("Block database migration complete") + return nil +} + +// readBlockTree reads the old block index bucket and constructs a mapping of +// each block to all child blocks. This mapping represents the full tree of +// blocks. +func readBlockTree(v1BlockIdxBucket database.Bucket) (map[chainhash.Hash][]*chainhash.Hash, error) { + childBlocksMap := make(map[chainhash.Hash][]*chainhash.Hash) + err := v1BlockIdxBucket.ForEach(func(_, blockRow []byte) error { + var header wire.BlockHeader + endOffset := blockHdrOffset + blockHdrSize + headerBytes := blockRow[blockHdrOffset:endOffset:endOffset] + err := header.Deserialize(bytes.NewReader(headerBytes)) + if err != nil { + return err + } + + blockHash := header.BlockHash() + childBlocksMap[header.PrevBlock] = + append(childBlocksMap[header.PrevBlock], &blockHash) + return nil + }) + return childBlocksMap, err +} + +// determineBlockHeights takes a map of block hashes to a slice of child hashes +// and uses it to compute the height for each block. The function assigns a +// height of 0 to the genesis hash and explores the tree of blocks +// breadth-first, assigning a height to every block with a path back to the +// genesis block. +func determineBlockHeights(childBlocksMap map[chainhash.Hash][]*chainhash.Hash) map[chainhash.Hash]uint32 { + blockHeights := make(map[chainhash.Hash]uint32) + queue := list.New() + + // The genesis block is included in childBlocksMap as a child of the zero + // hash because that is the value of the PrevBlock field in the genesis + // header. + for _, genesisHash := range childBlocksMap[zeroHash] { + blockHeights[*genesisHash] = 0 + queue.PushBack(genesisHash) + } + + for e := queue.Front(); e != nil; e = queue.Front() { + queue.Remove(e) + hash := e.Value.(*chainhash.Hash) + height := blockHeights[*hash] + + // For each block with this one as a parent, assign it a height and + // push to queue for future processing. + for _, childHash := range childBlocksMap[*hash] { + blockHeights[*childHash] = height + 1 + queue.PushBack(childHash) + } + } + return blockHeights +} diff --git a/blockchain/validate.go b/blockchain/validate.go index 27f04f6e..1beda89e 100644 --- a/blockchain/validate.go +++ b/blockchain/validate.go @@ -47,7 +47,7 @@ var ( // zeroHash is the zero value for a chainhash.Hash and is defined as // a package level variable to avoid the need to create a new instance // every time a check is needed. - zeroHash = &chainhash.Hash{} + zeroHash chainhash.Hash // block91842Hash is one of the two nodes which violate the rules // set forth in BIP0030. It is defined as a package level variable to @@ -63,7 +63,7 @@ var ( // isNullOutpoint determines whether or not a previous transaction output point // is set. func isNullOutpoint(outpoint *wire.OutPoint) bool { - if outpoint.Index == math.MaxUint32 && outpoint.Hash.IsEqual(zeroHash) { + if outpoint.Index == math.MaxUint32 && outpoint.Hash == zeroHash { return true } return false @@ -95,7 +95,7 @@ func IsCoinBaseTx(msgTx *wire.MsgTx) bool { // The previous output of a coin base must have a max value index and // a zero hash. prevOut := &msgTx.TxIn[0].PreviousOutPoint - if prevOut.Index != math.MaxUint32 || !prevOut.Hash.IsEqual(zeroHash) { + if prevOut.Index != math.MaxUint32 || prevOut.Hash != zeroHash { return false }