From 296fa0a5a05b1053662ee2208f044de7de7709f6 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 3 Feb 2017 12:13:53 -0600 Subject: [PATCH] blockchain: Convert to full block index in mem. This reworks the block index code such that it loads all of the headers in the main chain at startup and constructs the full block index accordingly. Since the full index from the current best tip all the way back to the genesis block is now guaranteed to be in memory, this also removes all code related to dynamically loading the nodes and updates some of the logic to take advantage of the fact traversing the block index can longer potentially fail. There are also more optimizations and simplifications that can be made in the future as a result of this. Due to removing all of the extra overhead of tracking the dynamic state, and ensuring the block node structs are aligned to eliminate extra padding, the end result of a fully populated block index now takes quite a bit less memory than the previous dynamically loaded version. The main downside is that it now takes a while to start whereas it was nearly instant before, however, it is much better to provide more efficient runtime operation since that is its ultimate purpose and the benefits far outweigh this downside. Some benefits are: - Since every block node is in memory, the recent code which reconstructs headers from block nodes means that all headers can always be served from memory which is important since the majority of the network has moved to header-based semantics - Several of the error paths can be removed since they are no longer necessary - It is no longer expensive to calculate CSV sequence locks or median times of blocks way in the past - It will be possible to create much more efficient iteration and simplified views of the overall index - The entire threshold state database cache can be removed since it is cheap to construct it from the full block index as needed An overview of the logic changes are as follows: - Move AncestorNode from blockIndex to blockNode and greatly simplify since it no longer has to deal with the possibility of dynamically loading nodes and related failures - Rename RelativeNode to RelativeAncestor, move to blockNode, and redefine in terms of AncestorNode - Move CalcPastMedianTime from blockIndex to blockNode and remove no longer necessary test for nil - Change calcSequenceLock to use Ancestor instead of RelativeAncestor since it reads more clearly --- blockchain/accept.go | 13 +- blockchain/blockindex.go | 458 +++++++++-------------------------- blockchain/blocklocator.go | 6 - blockchain/chain.go | 96 ++------ blockchain/chainio.go | 69 +++--- blockchain/difficulty.go | 39 +-- blockchain/process.go | 2 +- blockchain/thresholdstate.go | 42 +--- blockchain/validate.go | 26 +- blockchain/versionbits.go | 30 +-- mining/mining.go | 3 +- 11 files changed, 203 insertions(+), 581 deletions(-) diff --git a/blockchain/accept.go b/blockchain/accept.go index a53a1d08..3d32f8c0 100644 --- a/blockchain/accept.go +++ b/blockchain/accept.go @@ -26,17 +26,10 @@ import ( func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) (bool, error) { dryRun := flags&BFDryRun == BFDryRun - // Get a block node for the block previous to this one. Will be nil - // if this is the genesis block. - prevNode, err := b.index.PrevNodeFromBlock(block) - if err != nil { - log.Errorf("PrevNodeFromBlock: %v", err) - return false, err - } - // The height of this block is one more than the referenced previous // block. blockHeight := int32(0) + prevNode := b.index.LookupNode(&block.MsgBlock().Header.PrevBlock) if prevNode != nil { blockHeight = prevNode.height + 1 } @@ -44,7 +37,7 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) // The block must pass all of the validation rules which depend on the // position of the block within the block chain. - err = b.checkBlockContext(block, prevNode, flags) + err := b.checkBlockContext(block, prevNode, flags) if err != nil { return false, err } @@ -68,7 +61,7 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) // Create a new block node for the block and add it to the in-memory // block chain (could be either a side chain or the main chain). blockHeader := &block.MsgBlock().Header - newNode := newBlockNode(blockHeader, block.Hash(), blockHeight) + newNode := newBlockNode(blockHeader, blockHeight) if prevNode != nil { newNode.parent = prevNode newNode.height = blockHeight diff --git a/blockchain/blockindex.go b/blockchain/blockindex.go index 95bce829..a804c7fd 100644 --- a/blockchain/blockindex.go +++ b/blockchain/blockindex.go @@ -5,7 +5,6 @@ package blockchain import ( - "fmt" "math/big" "sort" "sync" @@ -15,37 +14,32 @@ import ( "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcutil" ) // blockNode represents a block within the block chain and is primarily used to // aid in selecting the best chain to be the main chain. The main chain is // stored into the block database. type blockNode struct { + // NOTE: Additions, deletions, or modifications to the order of the + // definitions in this struct should not be changed without considering + // how it affects alignment on 64-bit platforms. The current order is + // specifically crafted to result in minimal padding. There will be + // hundreds of thousands of these in memory, so a few extra bytes of + // padding adds up. + // parent is the parent block for this node. parent *blockNode - // children contains the child nodes for this node. Typically there - // will only be one, but sometimes there can be more than one and that - // is when the best chain selection algorithm is used. - children []*blockNode - // hash is the double sha 256 of the block. hash chainhash.Hash - // parentHash is the double sha 256 of the parent block. This is kept - // here over simply relying on parent.hash directly since block nodes - // are sparse and the parent node might not be in memory when its hash - // is needed. - parentHash chainhash.Hash - - // height is the position in the block chain. - height int32 - // workSum is the total amount of work in the chain up to and including // this node. workSum *big.Int + // height is the position in the block chain. + height int32 + // inMainChain denotes whether the block node is currently on the // the main chain or not. This is used to help find the common // ancestor when switching chains. @@ -62,14 +56,16 @@ type blockNode struct { merkleRoot chainhash.Hash } -// newBlockNode returns a new block node for the given block header. It is -// completely disconnected from the chain and the workSum value is just the work -// for the passed block. The work sum is updated accordingly when the node is -// inserted into a chain. -func newBlockNode(blockHeader *wire.BlockHeader, blockHash *chainhash.Hash, height int32) *blockNode { - node := blockNode{ - hash: *blockHash, - parentHash: blockHeader.PrevBlock, +// initBlockNode initializes a block node from the given header and height. The +// node is completely disconnected from the chain and the workSum value is just +// the work for the passed block. The work sum must be updated accordingly when +// the node is inserted into a chain. +// +// This function is NOT safe for concurrent access. It must only be called when +// initially creating a node. +func initBlockNode(node *blockNode, blockHeader *wire.BlockHeader, height int32) { + *node = blockNode{ + hash: blockHeader.BlockHash(), workSum: CalcWork(blockHeader.Bits), height: height, version: blockHeader.Version, @@ -78,6 +74,15 @@ func newBlockNode(blockHeader *wire.BlockHeader, blockHash *chainhash.Hash, heig timestamp: blockHeader.Timestamp.Unix(), merkleRoot: blockHeader.MerkleRoot, } +} + +// newBlockNode returns a new block node for the given block header. It is +// completely disconnected from the chain and the workSum value is just the work +// for the passed block. The work sum must be updated accordingly when the node +// is inserted into a chain. +func newBlockNode(blockHeader *wire.BlockHeader, height int32) *blockNode { + var node blockNode + initBlockNode(&node, blockHeader, height) return &node } @@ -86,9 +91,13 @@ func newBlockNode(blockHeader *wire.BlockHeader, blockHash *chainhash.Hash, heig // This function is safe for concurrent access. func (node *blockNode) Header() wire.BlockHeader { // No lock is needed because all accessed fields are immutable. + prevHash := zeroHash + if node.parent != nil { + prevHash = &node.parent.hash + } return wire.BlockHeader{ Version: node.version, - PrevBlock: node.parentHash, + PrevBlock: *prevHash, MerkleRoot: node.merkleRoot, Timestamp: time.Unix(node.timestamp, 0), Bits: node.bits, @@ -96,347 +105,50 @@ func (node *blockNode) Header() wire.BlockHeader { } } -// removeChildNode deletes node from the provided slice of child block -// nodes. It ensures the final pointer reference is set to nil to prevent -// potential memory leaks. The original slice is returned unmodified if node -// is invalid or not in the slice. -// -// This function MUST be called with the block index lock held (for writes). -func removeChildNode(children []*blockNode, node *blockNode) []*blockNode { - if node == nil { - return children - } - - // An indexing for loop is intentionally used over a range here as range - // does not reevaluate the slice on each iteration nor does it adjust - // the index for the modified slice. - for i := 0; i < len(children); i++ { - if children[i].hash.IsEqual(&node.hash) { - copy(children[i:], children[i+1:]) - children[len(children)-1] = nil - return children[:len(children)-1] - } - } - return children -} - -// blockIndex provides facilities for keeping track of an in-memory index of the -// block chain. Although the name block chain suggest a single chain of blocks, -// it is actually a tree-shaped structure where any node can have multiple -// children. However, there can only be one active branch which does indeed -// form a chain from the tip all the way back to the genesis block. -type blockIndex struct { - // The following fields are set when the instance is created and can't - // be changed afterwards, so there is no need to protect them with a - // separate mutex. - db database.DB - chainParams *chaincfg.Params - - sync.RWMutex - index map[chainhash.Hash]*blockNode - depNodes map[chainhash.Hash][]*blockNode -} - -// newBlockIndex returns a new empty instance of a block index. The index will -// be dynamically populated as block nodes are loaded from the database and -// manually added. -func newBlockIndex(db database.DB, chainParams *chaincfg.Params) *blockIndex { - return &blockIndex{ - db: db, - chainParams: chainParams, - index: make(map[chainhash.Hash]*blockNode), - depNodes: make(map[chainhash.Hash][]*blockNode), - } -} - -// HaveBlock returns whether or not the block index contains the provided hash. +// Ancestor returns the ancestor block node at the provided height by following +// the chain backwards from this node. The returned block will be nil when a +// height is requested that is after the height of the passed node or is less +// than zero. // // This function is safe for concurrent access. -func (bi *blockIndex) HaveBlock(hash *chainhash.Hash) bool { - bi.RLock() - _, hasBlock := bi.index[*hash] - bi.RUnlock() - return hasBlock -} - -// loadBlockNode loads the block identified by hash from the block database, -// creates a block node from it, and updates the block index accordingly. It is -// used mainly to dynamically load previous blocks from the database as they are -// needed to avoid needing to put the entire block index in memory. -// -// This function MUST be called with the block index lock held (for writes). -// The database transaction may be read-only. -func (bi *blockIndex) loadBlockNode(dbTx database.Tx, hash *chainhash.Hash) (*blockNode, error) { - // Load the block header and height from the db. - blockHeader, err := dbFetchHeaderByHash(dbTx, hash) - if err != nil { - return nil, err - } - blockHeight, err := dbFetchHeightByHash(dbTx, hash) - if err != nil { - return nil, err - } - - // Create the new block node for the block and set the work. - node := newBlockNode(blockHeader, hash, blockHeight) - node.inMainChain = true - - // Add the node to the chain. - // There are a few possibilities here: - // 1) This node is a child of an existing block node - // 2) This node is the parent of one or more nodes - // 3) Neither 1 or 2 is true which implies it's an orphan block and - // therefore is an error to insert into the chain - prevHash := &blockHeader.PrevBlock - if parentNode, ok := bi.index[*prevHash]; ok { - // Case 1 -- This node is a child of an existing block node. - // Update the node's work sum with the sum of the parent node's - // work sum and this node's work, append the node as a child of - // the parent node and set this node's parent to the parent - // node. - node.workSum = node.workSum.Add(parentNode.workSum, node.workSum) - parentNode.children = append(parentNode.children, node) - node.parent = parentNode - - } else if childNodes, ok := bi.depNodes[*hash]; ok { - // Case 2 -- This node is the parent of one or more nodes. - // Update the node's work sum by subtracting this node's work - // from the sum of its first child, and connect the node to all - // of its children. - node.workSum.Sub(childNodes[0].workSum, node.workSum) - for _, childNode := range childNodes { - childNode.parent = node - node.children = append(node.children, childNode) - } - - } else { - // Case 3 -- The node doesn't have a parent and is not the - // parent of another node. This means an arbitrary orphan block - // is trying to be loaded which is not allowed. - str := "loadBlockNode: attempt to insert orphan block %v" - return nil, AssertError(fmt.Sprintf(str, hash)) - } - - // Add the new node to the indices for faster lookups. - bi.index[*hash] = node - bi.depNodes[*prevHash] = append(bi.depNodes[*prevHash], node) - - return node, nil -} - -// PrevNodeFromBlock returns a block node for the block previous to the passed -// block (the passed block's parent). When it is already in the memory block -// chain, it simply returns it. Otherwise, it loads the previous block header -// from the block database, creates a new block node from it, and returns it. -// The returned node will be nil if the genesis block is passed. -// -// This function is safe for concurrent access. -func (bi *blockIndex) PrevNodeFromBlock(block *btcutil.Block) (*blockNode, error) { - // Genesis block. - prevHash := &block.MsgBlock().Header.PrevBlock - if prevHash.IsEqual(zeroHash) { - return nil, nil - } - - bi.Lock() - defer bi.Unlock() - - // Return the existing previous block node if it's already there. - if bn, ok := bi.index[*prevHash]; ok { - return bn, nil - } - - // Dynamically load the previous block from the block database, create - // a new block node for it, and update the memory chain accordingly. - var prevBlockNode *blockNode - err := bi.db.View(func(dbTx database.Tx) error { - var err error - prevBlockNode, err = bi.loadBlockNode(dbTx, prevHash) - return err - }) - return prevBlockNode, err -} - -// prevNodeFromNode returns a block node for the block previous to the -// passed block node (the passed block node's parent). When the node is already -// connected to a parent, it simply returns it. Otherwise, it loads the -// associated block from the database to obtain the previous hash and uses that -// to dynamically create a new block node and return it. The memory block -// chain is updated accordingly. The returned node will be nil if the genesis -// block is passed. -// -// This function MUST be called with the block index lock held (for writes). -func (bi *blockIndex) prevNodeFromNode(node *blockNode) (*blockNode, error) { - // Return the existing previous block node if it's already there. - if node.parent != nil { - return node.parent, nil - } - - // Genesis block. - if node.hash.IsEqual(bi.chainParams.GenesisHash) { - return nil, nil - } - - // Dynamically load the previous block from the block database, create - // a new block node for it, and update the memory chain accordingly. - var prevBlockNode *blockNode - err := bi.db.View(func(dbTx database.Tx) error { - var err error - prevBlockNode, err = bi.loadBlockNode(dbTx, &node.parentHash) - return err - }) - return prevBlockNode, err -} - -// PrevNodeFromNode returns a block node for the block previous to the -// passed block node (the passed block node's parent). When the node is already -// connected to a parent, it simply returns it. Otherwise, it loads the -// associated block from the database to obtain the previous hash and uses that -// to dynamically create a new block node and return it. The memory block -// chain is updated accordingly. The returned node will be nil if the genesis -// block is passed. -// -// This function is safe for concurrent access. -func (bi *blockIndex) PrevNodeFromNode(node *blockNode) (*blockNode, error) { - bi.Lock() - node, err := bi.prevNodeFromNode(node) - bi.Unlock() - return node, err -} - -// RelativeNode returns the ancestor block a relative 'distance' blocks before -// the passed anchor block. While iterating backwards through the chain, any -// block nodes which aren't in the memory chain are loaded in dynamically. -// -// This function is safe for concurrent access. -func (bi *blockIndex) RelativeNode(anchor *blockNode, distance uint32) (*blockNode, error) { - bi.Lock() - defer bi.Unlock() - - iterNode := anchor - err := bi.db.View(func(dbTx database.Tx) error { - // Walk backwards in the chian until we've gone 'distance' - // steps back. - var err error - for i := distance; i > 0; i-- { - switch { - // If the parent of this node has already been loaded - // into memory, then we can follow the link without - // hitting the database. - case iterNode.parent != nil: - iterNode = iterNode.parent - - // If this node is the genesis block, then we can't go - // back any further, so we exit immediately. - case iterNode.hash.IsEqual(bi.chainParams.GenesisHash): - return nil - - // Otherwise, load the block node from the database, - // pulling it into the memory cache in the processes. - default: - iterNode, err = bi.loadBlockNode(dbTx, - &iterNode.parentHash) - if err != nil { - return err - } - } - } - +func (node *blockNode) Ancestor(height int32) *blockNode { + if height < 0 || height > node.height { return nil - }) - if err != nil { - return nil, err } - return iterNode, nil + n := node + for ; n != nil && n.height != height; n = n.parent { + // Intentionally left blank + } + + return n } -// AncestorNode returns the ancestor block node at the provided height by -// following the chain backwards from the given node while dynamically loading -// any pruned nodes from the database and updating the memory block chain as -// needed. The returned block will be nil when a height is requested that is -// after the height of the passed node or is less than zero. +// RelativeAncestor returns the ancestor block node a relative 'distance' blocks +// before this node. This is equivalent to calling Ancestor with the node's +// height minus provided distance. // // This function is safe for concurrent access. -func (bi *blockIndex) AncestorNode(node *blockNode, height int32) (*blockNode, error) { - // Nothing to do if the requested height is outside of the valid range. - if height > node.height || height < 0 { - return nil, nil - } - - // Iterate backwards until the requested height is reached. - bi.Lock() - iterNode := node - for iterNode != nil && iterNode.height > height { - var err error - iterNode, err = bi.prevNodeFromNode(iterNode) - if err != nil { - break - } - } - bi.Unlock() - - return iterNode, nil -} - -// AddNode adds the provided node to the block index. Duplicate entries are not -// checked so it is up to caller to avoid adding them. -// -// This function is safe for concurrent access. -func (bi *blockIndex) AddNode(node *blockNode) { - bi.Lock() - bi.index[node.hash] = node - if prevHash := &node.parentHash; *prevHash != *zeroHash { - bi.depNodes[*prevHash] = append(bi.depNodes[*prevHash], node) - } - bi.Unlock() -} - -// LookupNode returns the block node identified by the provided hash. It will -// return nil if there is no entry for the hash. -// -// This function is safe for concurrent access. -func (bi *blockIndex) LookupNode(hash *chainhash.Hash) *blockNode { - bi.RLock() - node := bi.index[*hash] - bi.RUnlock() - return node +func (node *blockNode) RelativeAncestor(distance int32) *blockNode { + return node.Ancestor(node.height - distance) } // CalcPastMedianTime calculates the median time of the previous few blocks -// prior to, and including, the passed block node. +// prior to, and including, the block node. // // This function is safe for concurrent access. -func (bi *blockIndex) CalcPastMedianTime(startNode *blockNode) (time.Time, error) { - // Genesis block. - if startNode == nil { - return bi.chainParams.GenesisBlock.Header.Timestamp, nil - } - +func (node *blockNode) CalcPastMedianTime() time.Time { // Create a slice of the previous few block timestamps used to calculate // the median per the number defined by the constant medianTimeBlocks. timestamps := make([]int64, medianTimeBlocks) numNodes := 0 - iterNode := startNode - bi.Lock() + iterNode := node for i := 0; i < medianTimeBlocks && iterNode != nil; i++ { timestamps[i] = iterNode.timestamp numNodes++ - // Get the previous block node. This function is used over - // simply accessing iterNode.parent directly as it will - // dynamically create previous block nodes as needed. This - // helps allow only the pieces of the chain that are needed - // to remain in memory. - var err error - iterNode, err = bi.prevNodeFromNode(iterNode) - if err != nil { - bi.Unlock() - log.Errorf("prevNodeFromNode: %v", err) - return time.Time{}, err - } + iterNode = iterNode.parent } - bi.Unlock() // Prune the slice to the actual number of available timestamps which // will be fewer than desired near the beginning of the block chain @@ -457,5 +169,63 @@ func (bi *blockIndex) CalcPastMedianTime(startNode *blockNode) (time.Time, error // aware that should the medianTimeBlocks constant ever be changed to an // even number, this code will be wrong. medianTimestamp := timestamps[numNodes/2] - return time.Unix(medianTimestamp, 0), nil + return time.Unix(medianTimestamp, 0) +} + +// blockIndex provides facilities for keeping track of an in-memory index of the +// block chain. Although the name block chain suggests a single chain of +// blocks, it is actually a tree-shaped structure where any node can have +// multiple children. However, there can only be one active branch which does +// indeed form a chain from the tip all the way back to the genesis block. +type blockIndex struct { + // The following fields are set when the instance is created and can't + // be changed afterwards, so there is no need to protect them with a + // separate mutex. + db database.DB + chainParams *chaincfg.Params + + sync.RWMutex + index map[chainhash.Hash]*blockNode +} + +// newBlockIndex returns a new empty instance of a block index. The index will +// be dynamically populated as block nodes are loaded from the database and +// manually added. +func newBlockIndex(db database.DB, chainParams *chaincfg.Params) *blockIndex { + return &blockIndex{ + db: db, + chainParams: chainParams, + index: make(map[chainhash.Hash]*blockNode), + } +} + +// HaveBlock returns whether or not the block index contains the provided hash. +// +// This function is safe for concurrent access. +func (bi *blockIndex) HaveBlock(hash *chainhash.Hash) bool { + bi.RLock() + _, hasBlock := bi.index[*hash] + bi.RUnlock() + return hasBlock +} + +// LookupNode returns the block node identified by the provided hash. It will +// return nil if there is no entry for the hash. +// +// This function is safe for concurrent access. +func (bi *blockIndex) LookupNode(hash *chainhash.Hash) *blockNode { + bi.RLock() + node := bi.index[*hash] + bi.RUnlock() + return node +} + +// AddNode adds the provided node to the block index. Duplicate entries are not +// checked so it is up to caller to avoid adding them. +// +// This function is safe for concurrent access. +func (bi *blockIndex) AddNode(node *blockNode) { + bi.Lock() + bi.index[node.hash] = node + bi.Unlock() } diff --git a/blockchain/blocklocator.go b/blockchain/blocklocator.go index 86e987df..64184831 100644 --- a/blockchain/blocklocator.go +++ b/blockchain/blocklocator.go @@ -111,12 +111,6 @@ func (bi *blockIndex) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { // backwards along the side chain nodes to each block // height. if forkHeight != -1 && blockHeight > forkHeight { - // Intentionally use parent field instead of the - // PrevNodeFromNode function since we don't - // want to dynamically load nodes when building - // block locators. Side chain blocks should - // always be in memory already, and if they - // aren't for some reason it's ok to skip them. for iterNode != nil && blockHeight > iterNode.height { iterNode = iterNode.parent } diff --git a/blockchain/chain.go b/blockchain/chain.go index eaf8b49f..76316f02 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -193,10 +193,7 @@ func (b *BlockChain) DisableVerify(disable bool) { // // This function is safe for concurrent access. func (b *BlockChain) HaveBlock(hash *chainhash.Hash) (bool, error) { - b.chainLock.RLock() exists, err := b.blockExists(hash) - b.chainLock.RUnlock() - if err != nil { return false, err } @@ -378,15 +375,10 @@ func (b *BlockChain) calcSequenceLock(node *blockNode, tx *btcutil.Tx, utxoView // If we're performing block validation, then we need to query the BIP9 // state. if !csvSoftforkActive { - prevNode, err := b.index.PrevNodeFromNode(node) - if err != nil { - return nil, err - } - // Obtain the latest BIP9 version bits state for the // CSV-package soft-fork deployment. The adherence of sequence // locks depends on the current soft-fork state. - csvState, err := b.deploymentState(prevNode, chaincfg.DeploymentCSV) + csvState, err := b.deploymentState(node.parent, chaincfg.DeploymentCSV) if err != nil { return nil, err } @@ -439,26 +431,17 @@ func (b *BlockChain) calcSequenceLock(node *blockNode, tx *btcutil.Tx, utxoView continue case sequenceNum&wire.SequenceLockTimeIsSeconds == wire.SequenceLockTimeIsSeconds: // This input requires a relative time lock expressed - // in seconds before it can be spent. Therefore, we + // in seconds before it can be spent. Therefore, we // need to query for the block prior to the one in // which this input was included within so we can // compute the past median time for the block prior to // the one which included this referenced output. - // TODO: caching should be added to keep this speedy - inputDepth := uint32(node.height-inputHeight) + 1 - blockNode, err := b.index.RelativeNode(node, inputDepth) - if err != nil { - return sequenceLock, err - } - - // With all the necessary block headers loaded into - // memory, we can now finally calculate the MTP of the - // block prior to the one which included the output - // being spent. - medianTime, err := b.index.CalcPastMedianTime(blockNode) - if err != nil { - return sequenceLock, err + prevInputHeight := inputHeight - 1 + if prevInputHeight < 0 { + prevInputHeight = 0 } + blockNode := node.Ancestor(prevInputHeight) + medianTime := blockNode.CalcPastMedianTime() // Time based relative time-locks as defined by BIP 68 // have a time granularity of RelativeLockSeconds, so @@ -535,12 +518,6 @@ func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List attachNodes.PushFront(ancestor) } - // TODO(davec): Use prevNodeFromNode function in case the requested - // node is further back than the what is in memory. This shouldn't - // happen in the normal course of operation, but the ability to fetch - // input transactions of arbitrary blocks will likely to be exposed at - // some point and that could lead to an issue here. - // Start from the end of the main chain and work backwards until the // common ancestor adding each block to the list of nodes to detach from // the main chain. @@ -609,12 +586,6 @@ func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block, view *U } } - // Calculate the median time for the block. - medianTime, err := b.index.CalcPastMedianTime(node) - if err != nil { - return err - } - // Generate a new best state snapshot that will be used to update the // database and later memory if all database updates are successful. b.stateLock.RLock() @@ -624,10 +595,10 @@ func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block, view *U blockSize := uint64(block.MsgBlock().SerializeSize()) blockWeight := uint64(GetBlockWeight(block)) state := newBestState(node, blockSize, blockWeight, numTxns, - curTotalTxns+numTxns, medianTime) + curTotalTxns+numTxns, node.CalcPastMedianTime()) // Atomically insert info into the database. - err = b.db.Update(func(dbTx database.Tx) error { + err := b.db.Update(func(dbTx database.Tx) error { // Update best block state. err := dbPutBestState(dbTx, state, node.workSum) if err != nil { @@ -718,24 +689,10 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block *btcutil.Block, view "block at the end of the main chain") } - // Get the previous block node. This function is used over simply - // accessing node.parent directly as it will dynamically create previous - // block nodes as needed. This helps allow only the pieces of the chain - // that are needed to remain in memory. - prevNode, err := b.index.PrevNodeFromNode(node) - if err != nil { - return err - } - - // Calculate the median time for the previous block. - medianTime, err := b.index.CalcPastMedianTime(prevNode) - if err != nil { - return err - } - // Load the previous block since some details for it are needed below. + prevNode := node.parent var prevBlock *btcutil.Block - err = b.db.View(func(dbTx database.Tx) error { + err := b.db.View(func(dbTx database.Tx) error { var err error prevBlock, err = dbFetchBlockByHash(dbTx, &prevNode.hash) return err @@ -754,7 +711,7 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block *btcutil.Block, view blockWeight := uint64(GetBlockWeight(prevBlock)) newTotalTxns := curTotalTxns - uint64(len(block.MsgBlock().Transactions)) state := newBestState(prevNode, blockSize, blockWeight, numTxns, - newTotalTxns, medianTime) + newTotalTxns, prevNode.CalcPastMedianTime()) err = b.db.Update(func(dbTx database.Tx) error { // Update best block state. @@ -1026,16 +983,12 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags } } - // Log the point where the chain forked. + // Log the point where the chain forked and old and new best chain + // heads. firstAttachNode := attachNodes.Front().Value.(*blockNode) - forkNode, err := b.index.PrevNodeFromNode(firstAttachNode) - if err == nil { - log.Infof("REORGANIZE: Chain forks at %v", forkNode.hash) - } - - // Log the old and new best chain heads. firstDetachNode := detachNodes.Front().Value.(*blockNode) lastAttachNode := attachNodes.Back().Value.(*blockNode) + log.Infof("REORGANIZE: Chain forks at %v", firstAttachNode.parent.hash) log.Infof("REORGANIZE: Old best chain head was %v", firstDetachNode.hash) log.Infof("REORGANIZE: New best chain head is %v", lastAttachNode.hash) @@ -1065,12 +1018,13 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla // We are extending the main (best) chain with a new block. This is the // most common case. - if node.parentHash.IsEqual(&b.bestNode.hash) { + parentHash := &block.MsgBlock().Header.PrevBlock + if parentHash.IsEqual(&b.bestNode.hash) { // Perform several checks to verify the block can be connected // to the main chain without violating any rules and without // actually connecting the block. view := NewUtxoViewpoint() - view.SetBestHash(&node.parentHash) + view.SetBestHash(parentHash) stxos := make([]spentTxOut, 0, countSpentOutputs(block)) if !fastAdd { err := b.checkConnectBlock(node, block, view, &stxos) @@ -1105,11 +1059,6 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla return false, err } - // Connect the parent node to this node. - if node.parent != nil { - node.parent.children = append(node.parent.children, node) - } - return true, nil } if fastAdd { @@ -1124,18 +1073,13 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla b.index.index[node.hash] = node b.index.Unlock() - // Connect the parent node to this node. + // Mark node as in a side chain. node.inMainChain = false - node.parent.children = append(node.parent.children, node) // Disconnect it from the parent node when the function returns when // running in dry run mode. if dryRun { defer func() { - children := node.parent.children - children = removeChildNode(children, node) - node.parent.children = children - b.index.Lock() delete(b.index.index, node.hash) b.index.Unlock() @@ -1159,7 +1103,7 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla } // Log information about how the block is forking the chain. - if fork.hash.IsEqual(&node.parent.hash) { + if fork.hash.IsEqual(parentHash) { log.Infof("FORK: Block %v forks the chain at height %d"+ "/block %v, but does not cause a reorganize", node.hash, fork.height, fork.hash) diff --git a/blockchain/chainio.go b/blockchain/chainio.go index 92987983..4c6afda1 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1081,7 +1081,7 @@ func (b *BlockChain) createChainState() error { // Create a new node from the genesis block and set it as the best node. genesisBlock := btcutil.NewBlock(b.chainParams.GenesisBlock) header := &genesisBlock.MsgBlock().Header - node := newBlockNode(header, genesisBlock.Hash(), 0) + node := newBlockNode(header, 0) node.inMainChain = true b.bestNode = node @@ -1168,6 +1168,44 @@ func (b *BlockChain) initChainState() error { return err } + // Load all of the headers from the data for the known best + // chain and construct the block index accordingly. Since the + // number of nodes are already known, perform a single alloc + // for them versus a whole bunch of little ones to reduce + // pressure on the GC. + log.Infof("Loading block index. This might take a while...") + bestHeight := int32(state.height) + blockNodes := make([]blockNode, bestHeight+1) + for height := int32(0); height <= bestHeight; height++ { + header, err := dbFetchHeaderByHeight(dbTx, height) + if err != nil { + return err + } + + // Initialize the block node for the block, connect it, + // and add it to the block index. + node := &blockNodes[height] + initBlockNode(node, header, height) + if parent := b.bestNode; parent != nil { + node.parent = parent + node.workSum = node.workSum.Add(parent.workSum, + node.workSum) + } + node.inMainChain = true + b.index.AddNode(node) + + // This node is now the end of the best chain. + b.bestNode = node + } + + // Ensure the resulting best node matches the stored best state + // hash. + if b.bestNode.hash != state.hash { + return AssertError(fmt.Sprintf("initChainState: block "+ + "index chain tip %s does not match stored "+ + "best state %s", b.bestNode.hash, state.hash)) + } + // Load the raw block bytes for the best block. blockBytes, err := dbTx.FetchBlock(&state.hash) if err != nil { @@ -1179,29 +1217,12 @@ func (b *BlockChain) initChainState() error { return err } - // Create a new node and set it as the best node. The preceding - // nodes will be loaded on demand as needed. - header := &block.Header - node := newBlockNode(header, &state.hash, int32(state.height)) - node.inMainChain = true - node.workSum = state.workSum - b.bestNode = node - - // Add the new node to the block index. - b.index.AddNode(node) - - // Calculate the median time for the block. - medianTime, err := b.index.CalcPastMedianTime(node) - if err != nil { - return err - } - // Initialize the state related to the best block. blockSize := uint64(len(blockBytes)) blockWeight := uint64(GetBlockWeight(btcutil.NewBlock(&block))) numTxns := uint64(len(block.Transactions)) b.stateSnapshot = newBestState(b.bestNode, blockSize, blockWeight, - numTxns, state.totalTxns, medianTime) + numTxns, state.totalTxns, b.bestNode.CalcPastMedianTime()) isStateInitialized = true return nil @@ -1906,19 +1927,11 @@ func (b *BlockChain) initThresholdCaches() error { "change. This might take a while...") } - // Get the previous block node. This function is used over simply - // accessing b.bestNode.parent directly as it will dynamically create - // previous block nodes as needed. This helps allow only the pieces of - // the chain that are needed to remain in memory. - prevNode, err := b.index.PrevNodeFromNode(b.bestNode) - if err != nil { - return err - } - // Initialize the warning and deployment caches by calculating the // threshold state for each of them. This will ensure the caches are // populated and any states that needed to be recalculated due to // definition changes is done now. + prevNode := b.bestNode.parent for bit := uint32(0); bit < vbNumBits; bit++ { checker := bitConditionChecker{bit: bit, chain: b} cache := &b.warningCaches[bit] diff --git a/blockchain/difficulty.go b/blockchain/difficulty.go index 755e0979..aa8b3768 100644 --- a/blockchain/difficulty.go +++ b/blockchain/difficulty.go @@ -194,24 +194,14 @@ func (b *BlockChain) calcEasiestDifficulty(bits uint32, duration time.Duration) // did not have the special testnet minimum difficulty rule applied. // // This function MUST be called with the chain state lock held (for writes). -func (b *BlockChain) findPrevTestNetDifficulty(startNode *blockNode) (uint32, error) { +func (b *BlockChain) findPrevTestNetDifficulty(startNode *blockNode) uint32 { // Search backwards through the chain for the last block without // the special rule applied. iterNode := startNode for iterNode != nil && iterNode.height%b.blocksPerRetarget != 0 && iterNode.bits == b.chainParams.PowLimitBits { - // Get the previous block node. This function is used over - // simply accessing iterNode.parent directly as it will - // dynamically create previous block nodes as needed. This - // helps allow only the pieces of the chain that are needed - // to remain in memory. - var err error - iterNode, err = b.index.PrevNodeFromNode(iterNode) - if err != nil { - log.Errorf("PrevNodeFromNode: %v", err) - return 0, err - } + iterNode = iterNode.parent } // Return the found difficulty or the minimum difficulty if no @@ -220,7 +210,7 @@ func (b *BlockChain) findPrevTestNetDifficulty(startNode *blockNode) (uint32, er if iterNode != nil { lastBits = iterNode.bits } - return lastBits, nil + return lastBits } // calcNextRequiredDifficulty calculates the required difficulty for the block @@ -228,8 +218,6 @@ func (b *BlockChain) findPrevTestNetDifficulty(startNode *blockNode) (uint32, er // This function differs from the exported CalcNextRequiredDifficulty in that // the exported version uses the current best chain as the previous block node // while this function accepts any block node. -// -// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) calcNextRequiredDifficulty(lastNode *blockNode, newBlockTime time.Time) (uint32, error) { // Genesis block. if lastNode == nil { @@ -255,11 +243,7 @@ func (b *BlockChain) calcNextRequiredDifficulty(lastNode *blockNode, newBlockTim // The block was mined within the desired timeframe, so // return the difficulty for the last block which did // not have the special minimum difficulty rule applied. - prevBits, err := b.findPrevTestNetDifficulty(lastNode) - if err != nil { - return 0, err - } - return prevBits, nil + return b.findPrevTestNetDifficulty(lastNode), nil } // For the main network (or any unrecognized networks), simply @@ -269,20 +253,7 @@ func (b *BlockChain) calcNextRequiredDifficulty(lastNode *blockNode, newBlockTim // Get the block node at the previous retarget (targetTimespan days // worth of blocks). - firstNode := lastNode - for i := int32(0); i < b.blocksPerRetarget-1 && firstNode != nil; i++ { - // Get the previous block node. This function is used over - // simply accessing firstNode.parent directly as it will - // dynamically create previous block nodes as needed. This - // helps allow only the pieces of the chain that are needed - // to remain in memory. - var err error - firstNode, err = b.index.PrevNodeFromNode(firstNode) - if err != nil { - return 0, err - } - } - + firstNode := lastNode.RelativeAncestor(b.blocksPerRetarget - 1) if firstNode == nil { return 0, AssertError("unable to obtain previous retarget block") } diff --git a/blockchain/process.go b/blockchain/process.go index 9bc68623..82d4c403 100644 --- a/blockchain/process.go +++ b/blockchain/process.go @@ -40,7 +40,7 @@ const ( // blockExists determines whether a block with the given hash exists either in // the main chain or any side chains. // -// This function MUST be called with the chain state lock held (for reads). +// This function is safe for concurrent access. func (b *BlockChain) blockExists(hash *chainhash.Hash) (bool, error) { // Check block index first (could be main chain or side chain blocks). if b.index.HaveBlock(hash) { diff --git a/blockchain/thresholdstate.go b/blockchain/thresholdstate.go index f9b736ea..e67d0e5c 100644 --- a/blockchain/thresholdstate.go +++ b/blockchain/thresholdstate.go @@ -157,12 +157,8 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit // Get the ancestor that is the last block of the previous confirmation // window in order to get its threshold state. This can be done because // the state is the same for all blocks within a given window. - var err error - prevNode, err = b.index.AncestorNode(prevNode, prevNode.height- + prevNode = prevNode.Ancestor(prevNode.height - (prevNode.height+1)%confirmationWindow) - if err != nil { - return ThresholdFailed, err - } // Iterate backwards through each of the previous confirmation windows // to find the most recently cached threshold state. @@ -176,10 +172,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit // The start and expiration times are based on the median block // time, so calculate it now. - medianTime, err := b.index.CalcPastMedianTime(prevNode) - if err != nil { - return ThresholdFailed, err - } + medianTime := prevNode.CalcPastMedianTime() // The state is simply defined if the start time hasn't been // been reached yet. @@ -194,11 +187,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit // Get the ancestor that is the last block of the previous // confirmation window. - prevNode, err = b.index.AncestorNode(prevNode, prevNode.height- - confirmationWindow) - if err != nil { - return ThresholdFailed, err - } + prevNode = prevNode.RelativeAncestor(confirmationWindow) } // Start with the threshold state for the most recent confirmation @@ -223,10 +212,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit case ThresholdDefined: // The deployment of the rule change fails if it expires // before it is accepted and locked in. - medianTime, err := b.index.CalcPastMedianTime(prevNode) - if err != nil { - return ThresholdFailed, err - } + medianTime := prevNode.CalcPastMedianTime() medianTimeUnix := uint64(medianTime.Unix()) if medianTimeUnix >= checker.EndTime() { state = ThresholdFailed @@ -243,10 +229,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit case ThresholdStarted: // The deployment of the rule change fails if it expires // before it is accepted and locked in. - medianTime, err := b.index.CalcPastMedianTime(prevNode) - if err != nil { - return ThresholdFailed, err - } + medianTime := prevNode.CalcPastMedianTime() if uint64(medianTime.Unix()) >= checker.EndTime() { state = ThresholdFailed break @@ -266,16 +249,8 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit count++ } - // Get the previous block node. This function - // is used over simply accessing countNode.parent - // directly as it will dynamically create - // previous block nodes as needed. This helps - // allow only the pieces of the chain that are - // needed to remain in memory. - countNode, err = b.index.PrevNodeFromNode(countNode) - if err != nil { - return ThresholdFailed, err - } + // Get the previous block node. + countNode = countNode.parent } // The state is locked in if the number of blocks in the @@ -341,8 +316,7 @@ func (b *BlockChain) IsDeploymentActive(deploymentID uint32) (bool, error) { // AFTER the passed node. // // This function MUST be called with the chain state lock held (for writes). -func (b *BlockChain) deploymentState(prevNode *blockNode, - deploymentID uint32) (ThresholdState, error) { +func (b *BlockChain) deploymentState(prevNode *blockNode, deploymentID uint32) (ThresholdState, error) { if deploymentID > uint32(len(b.chainParams.Deployments)) { return ThresholdFailed, DeploymentError(deploymentID) } diff --git a/blockchain/validate.go b/blockchain/validate.go index 58fcf973..01c31fee 100644 --- a/blockchain/validate.go +++ b/blockchain/validate.go @@ -668,11 +668,7 @@ func (b *BlockChain) checkBlockHeaderContext(header *wire.BlockHeader, prevNode // Ensure the timestamp for the block header is after the // median time of the last several blocks (medianTimeBlocks). - medianTime, err := b.index.CalcPastMedianTime(prevNode) - if err != nil { - log.Errorf("CalcPastMedianTime: %v", err) - return err - } + medianTime := prevNode.CalcPastMedianTime() if !header.Timestamp.After(medianTime) { str := "block timestamp of %v is not after expected %v" str = fmt.Sprintf(str, header.Timestamp, medianTime) @@ -762,12 +758,7 @@ func (b *BlockChain) checkBlockContext(block *btcutil.Block, prevNode *blockNode // timestamps for all lock-time based checks. blockTime := header.Timestamp if csvState == ThresholdActive { - medianTime, err := b.index.CalcPastMedianTime(prevNode) - if err != nil { - return err - } - - blockTime = medianTime + blockTime = prevNode.CalcPastMedianTime() } // The height of this block is one more than the referenced @@ -1015,10 +1006,11 @@ func (b *BlockChain) checkConnectBlock(node *blockNode, block *btcutil.Block, vi } // Ensure the view is for the node being checked. - if !view.BestHash().IsEqual(&node.parentHash) { + parentHash := &block.MsgBlock().Header.PrevBlock + if !view.BestHash().IsEqual(parentHash) { return AssertError(fmt.Sprintf("inconsistent view when "+ "checking block connection: best hash is %v instead "+ - "of expected %v", view.BestHash(), node.hash)) + "of expected %v", view.BestHash(), parentHash)) } // BIP0030 added a rule to prevent blocks which contain duplicate @@ -1200,10 +1192,7 @@ func (b *BlockChain) checkConnectBlock(node *blockNode, block *btcutil.Block, vi // We obtain the MTP of the *previous* block in order to // determine if transactions in the current block are final. - medianTime, err := b.index.CalcPastMedianTime(node.parent) - if err != nil { - return err - } + medianTime := node.parent.CalcPastMedianTime() // Additionally, if the CSV soft-fork package is now active, // then we also enforce the relative sequence number based @@ -1268,8 +1257,7 @@ func (b *BlockChain) CheckConnectBlock(block *btcutil.Block) error { defer b.chainLock.Unlock() prevNode := b.bestNode - newNode := newBlockNode(&block.MsgBlock().Header, block.Hash(), - prevNode.height+1) + newNode := newBlockNode(&block.MsgBlock().Header, prevNode.height+1) newNode.parent = prevNode newNode.workSum.Add(prevNode.workSum, newNode.workSum) diff --git a/blockchain/versionbits.go b/blockchain/versionbits.go index b58ae9f7..3ddf11c4 100644 --- a/blockchain/versionbits.go +++ b/blockchain/versionbits.go @@ -113,15 +113,7 @@ func (c bitConditionChecker) Condition(node *blockNode) (bool, error) { return false, nil } - // Get the previous block node. This function is used over simply - // accessing node.parent directly as it will dynamically create previous - // block nodes as needed. This helps allow only the pieces of the chain - // that are needed to remain in memory. - prevNode, err := c.chain.index.PrevNodeFromNode(node) - if err != nil { - return false, err - } - expectedVersion, err := c.chain.calcNextBlockVersion(prevNode) + expectedVersion, err := c.chain.calcNextBlockVersion(node.parent) if err != nil { return false, err } @@ -244,21 +236,12 @@ func (b *BlockChain) CalcNextBlockVersion() (int32, error) { // // This function MUST be called with the chain state lock held (for writes) func (b *BlockChain) warnUnknownRuleActivations(node *blockNode) error { - // Get the previous block node. This function is used over simply - // accessing node.parent directly as it will dynamically create previous - // block nodes as needed. This helps allow only the pieces of the chain - // that are needed to remain in memory. - prevNode, err := b.index.PrevNodeFromNode(node) - if err != nil { - return err - } - // Warn if any unknown new rules are either about to activate or have // already been activated. for bit := uint32(0); bit < vbNumBits; bit++ { checker := bitConditionChecker{bit: bit, chain: b} cache := &b.warningCaches[bit] - state, err := b.thresholdState(prevNode, checker, cache) + state, err := b.thresholdState(node.parent, checker, cache) if err != nil { return err } @@ -305,14 +288,7 @@ func (b *BlockChain) warnUnknownVersions(node *blockNode) error { numUpgraded++ } - // Get the previous block node. This function is used over - // simply accessing node.parent directly as it will dynamically - // create previous block nodes as needed. This helps allow only - // the pieces of the chain that are needed to remain in memory. - node, err = b.index.PrevNodeFromNode(node) - if err != nil { - return err - } + node = node.parent } if numUpgraded > unknownVerWarnNum { log.Warn("Unknown block versions are being mined, so new " + diff --git a/mining/mining.go b/mining/mining.go index c0b3c9d3..37158940 100644 --- a/mining/mining.go +++ b/mining/mining.go @@ -445,7 +445,6 @@ func NewBlkTmplGenerator(policy *Policy, params *chaincfg.Params, func (g *BlkTmplGenerator) NewBlockTemplate(payToAddress btcutil.Address) (*BlockTemplate, error) { // Extend the most recently known best block. best := g.chain.BestSnapshot() - prevHash := &best.Hash nextBlockHeight := best.Height + 1 // Create a standard coinbase transaction paying to the provided @@ -864,7 +863,7 @@ mempoolLoop: var msgBlock wire.MsgBlock msgBlock.Header = wire.BlockHeader{ Version: nextBlockVersion, - PrevBlock: *prevHash, + PrevBlock: best.Hash, MerkleRoot: *merkles[len(merkles)-1], Timestamp: ts, Bits: reqDifficulty,