From 1ecfea49282c0dd2f23992fb1431b51f8329eda4 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 25 Apr 2016 04:12:09 -0500 Subject: [PATCH] blockchain: Refactor main block index logic. This refactors the block index logic into a separate struct and introduces an individual lock for it so it can be queried independent of the chain lock. --- blockchain/accept.go | 6 +- blockchain/blockindex.go | 465 +++++++++++++++++++++++++++++++++++ blockchain/blocklocator.go | 26 +- blockchain/chain.go | 412 ++----------------------------- blockchain/chainio.go | 12 +- blockchain/difficulty.go | 6 +- blockchain/process.go | 6 +- blockchain/thresholdstate.go | 16 +- blockchain/validate.go | 4 +- blockchain/versionbits.go | 8 +- 10 files changed, 529 insertions(+), 432 deletions(-) create mode 100644 blockchain/blockindex.go diff --git a/blockchain/accept.go b/blockchain/accept.go index 08828aae..a53a1d08 100644 --- a/blockchain/accept.go +++ b/blockchain/accept.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2016 The btcsuite developers +// Copyright (c) 2013-2017 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -28,9 +28,9 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) // Get a block node for the block previous to this one. Will be nil // if this is the genesis block. - prevNode, err := b.getPrevNodeFromBlock(block) + prevNode, err := b.index.PrevNodeFromBlock(block) if err != nil { - log.Errorf("getPrevNodeFromBlock: %v", err) + log.Errorf("PrevNodeFromBlock: %v", err) return false, err } diff --git a/blockchain/blockindex.go b/blockchain/blockindex.go new file mode 100644 index 00000000..c0bc7626 --- /dev/null +++ b/blockchain/blockindex.go @@ -0,0 +1,465 @@ +// Copyright (c) 2015-2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "fmt" + "math/big" + "sort" + "sync" + "time" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/database" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" +) + +// blockNode represents a block within the block chain and is primarily used to +// aid in selecting the best chain to be the main chain. The main chain is +// stored into the block database. +type blockNode struct { + // parent is the parent block for this node. + parent *blockNode + + // children contains the child nodes for this node. Typically there + // will only be one, but sometimes there can be more than one and that + // is when the best chain selection algorithm is used. + children []*blockNode + + // hash is the double sha 256 of the block. + hash *chainhash.Hash + + // parentHash is the double sha 256 of the parent block. This is kept + // here over simply relying on parent.hash directly since block nodes + // are sparse and the parent node might not be in memory when its hash + // is needed. + parentHash *chainhash.Hash + + // height is the position in the block chain. + height int32 + + // workSum is the total amount of work in the chain up to and including + // this node. + workSum *big.Int + + // inMainChain denotes whether the block node is currently on the + // the main chain or not. This is used to help find the common + // ancestor when switching chains. + inMainChain bool + + // Some fields from block headers to aid in best chain selection and + // reconstructing headers from memory. These must be treated as + // immutable and are intentionally ordered to avoid padding on 64-bit + // platforms. + version int32 + bits uint32 + nonce uint32 + timestamp int64 + merkleRoot chainhash.Hash +} + +// newBlockNode returns a new block node for the given block header. It is +// completely disconnected from the chain and the workSum value is just the work +// for the passed block. The work sum is updated accordingly when the node is +// inserted into a chain. +func newBlockNode(blockHeader *wire.BlockHeader, blockHash *chainhash.Hash, height int32) *blockNode { + // Make a copy of the hash so the node doesn't keep a reference to part + // of the full block/block header preventing it from being garbage + // collected. + prevHash := blockHeader.PrevBlock + node := blockNode{ + hash: blockHash, + parentHash: &prevHash, + workSum: CalcWork(blockHeader.Bits), + height: height, + version: blockHeader.Version, + bits: blockHeader.Bits, + nonce: blockHeader.Nonce, + timestamp: blockHeader.Timestamp.Unix(), + merkleRoot: blockHeader.MerkleRoot, + } + return &node +} + +// Header constructs a block header from the node and returns it. +// +// This function is safe for concurrent access. +func (node *blockNode) Header() wire.BlockHeader { + // No lock is needed because all accessed fields are immutable. + return wire.BlockHeader{ + Version: node.version, + PrevBlock: *node.parentHash, + MerkleRoot: node.merkleRoot, + Timestamp: time.Unix(node.timestamp, 0), + Bits: node.bits, + Nonce: node.nonce, + } +} + +// removeChildNode deletes node from the provided slice of child block +// nodes. It ensures the final pointer reference is set to nil to prevent +// potential memory leaks. The original slice is returned unmodified if node +// is invalid or not in the slice. +// +// This function MUST be called with the block index lock held (for writes). +func removeChildNode(children []*blockNode, node *blockNode) []*blockNode { + if node == nil { + return children + } + + // An indexing for loop is intentionally used over a range here as range + // does not reevaluate the slice on each iteration nor does it adjust + // the index for the modified slice. + for i := 0; i < len(children); i++ { + if children[i].hash.IsEqual(node.hash) { + copy(children[i:], children[i+1:]) + children[len(children)-1] = nil + return children[:len(children)-1] + } + } + return children +} + +// blockIndex provides facilities for keeping track of an in-memory index of the +// block chain. Although the name block chain suggest a single chain of blocks, +// it is actually a tree-shaped structure where any node can have multiple +// children. However, there can only be one active branch which does indeed +// form a chain from the tip all the way back to the genesis block. +type blockIndex struct { + // The following fields are set when the instance is created and can't + // be changed afterwards, so there is no need to protect them with a + // separate mutex. + db database.DB + chainParams *chaincfg.Params + + sync.RWMutex + index map[chainhash.Hash]*blockNode + depNodes map[chainhash.Hash][]*blockNode +} + +// newBlockIndex returns a new empty instance of a block index. The index will +// be dynamically populated as block nodes are loaded from the database and +// manually added. +func newBlockIndex(db database.DB, chainParams *chaincfg.Params) *blockIndex { + return &blockIndex{ + db: db, + chainParams: chainParams, + index: make(map[chainhash.Hash]*blockNode), + depNodes: make(map[chainhash.Hash][]*blockNode), + } +} + +// HaveBlock returns whether or not the block index contains the provided hash. +// +// This function is safe for concurrent access. +func (bi *blockIndex) HaveBlock(hash *chainhash.Hash) bool { + bi.RLock() + _, hasBlock := bi.index[*hash] + bi.RUnlock() + return hasBlock +} + +// loadBlockNode loads the block identified by hash from the block database, +// creates a block node from it, and updates the block index accordingly. It is +// used mainly to dynamically load previous blocks from the database as they are +// needed to avoid needing to put the entire block index in memory. +// +// This function MUST be called with the block index lock held (for writes). +// The database transaction may be read-only. +func (bi *blockIndex) loadBlockNode(dbTx database.Tx, hash *chainhash.Hash) (*blockNode, error) { + // Load the block header and height from the db. + blockHeader, err := dbFetchHeaderByHash(dbTx, hash) + if err != nil { + return nil, err + } + blockHeight, err := dbFetchHeightByHash(dbTx, hash) + if err != nil { + return nil, err + } + + // Create the new block node for the block and set the work. + node := newBlockNode(blockHeader, hash, blockHeight) + node.inMainChain = true + + // Add the node to the chain. + // There are a few possibilities here: + // 1) This node is a child of an existing block node + // 2) This node is the parent of one or more nodes + // 3) Neither 1 or 2 is true which implies it's an orphan block and + // therefore is an error to insert into the chain + prevHash := &blockHeader.PrevBlock + if parentNode, ok := bi.index[*prevHash]; ok { + // Case 1 -- This node is a child of an existing block node. + // Update the node's work sum with the sum of the parent node's + // work sum and this node's work, append the node as a child of + // the parent node and set this node's parent to the parent + // node. + node.workSum = node.workSum.Add(parentNode.workSum, node.workSum) + parentNode.children = append(parentNode.children, node) + node.parent = parentNode + + } else if childNodes, ok := bi.depNodes[*hash]; ok { + // Case 2 -- This node is the parent of one or more nodes. + // Update the node's work sum by subtracting this node's work + // from the sum of its first child, and connect the node to all + // of its children. + node.workSum.Sub(childNodes[0].workSum, node.workSum) + for _, childNode := range childNodes { + childNode.parent = node + node.children = append(node.children, childNode) + } + + } else { + // Case 3 -- The node doesn't have a parent and is not the + // parent of another node. This means an arbitrary orphan block + // is trying to be loaded which is not allowed. + str := "loadBlockNode: attempt to insert orphan block %v" + return nil, AssertError(fmt.Sprintf(str, hash)) + } + + // Add the new node to the indices for faster lookups. + bi.index[*hash] = node + bi.depNodes[*prevHash] = append(bi.depNodes[*prevHash], node) + + return node, nil +} + +// PrevNodeFromBlock returns a block node for the block previous to the passed +// block (the passed block's parent). When it is already in the memory block +// chain, it simply returns it. Otherwise, it loads the previous block header +// from the block database, creates a new block node from it, and returns it. +// The returned node will be nil if the genesis block is passed. +// +// This function is safe for concurrent access. +func (bi *blockIndex) PrevNodeFromBlock(block *btcutil.Block) (*blockNode, error) { + // Genesis block. + prevHash := &block.MsgBlock().Header.PrevBlock + if prevHash.IsEqual(zeroHash) { + return nil, nil + } + + bi.Lock() + defer bi.Unlock() + + // Return the existing previous block node if it's already there. + if bn, ok := bi.index[*prevHash]; ok { + return bn, nil + } + + // Dynamically load the previous block from the block database, create + // a new block node for it, and update the memory chain accordingly. + var prevBlockNode *blockNode + err := bi.db.View(func(dbTx database.Tx) error { + var err error + prevBlockNode, err = bi.loadBlockNode(dbTx, prevHash) + return err + }) + return prevBlockNode, err +} + +// prevNodeFromNode returns a block node for the block previous to the +// passed block node (the passed block node's parent). When the node is already +// connected to a parent, it simply returns it. Otherwise, it loads the +// associated block from the database to obtain the previous hash and uses that +// to dynamically create a new block node and return it. The memory block +// chain is updated accordingly. The returned node will be nil if the genesis +// block is passed. +// +// This function MUST be called with the block index lock held (for writes). +func (bi *blockIndex) prevNodeFromNode(node *blockNode) (*blockNode, error) { + // Return the existing previous block node if it's already there. + if node.parent != nil { + return node.parent, nil + } + + // Genesis block. + if node.hash.IsEqual(bi.chainParams.GenesisHash) { + return nil, nil + } + + // Dynamically load the previous block from the block database, create + // a new block node for it, and update the memory chain accordingly. + var prevBlockNode *blockNode + err := bi.db.View(func(dbTx database.Tx) error { + var err error + prevBlockNode, err = bi.loadBlockNode(dbTx, node.parentHash) + return err + }) + return prevBlockNode, err +} + +// PrevNodeFromNode returns a block node for the block previous to the +// passed block node (the passed block node's parent). When the node is already +// connected to a parent, it simply returns it. Otherwise, it loads the +// associated block from the database to obtain the previous hash and uses that +// to dynamically create a new block node and return it. The memory block +// chain is updated accordingly. The returned node will be nil if the genesis +// block is passed. +// +// This function is safe for concurrent access. +func (bi *blockIndex) PrevNodeFromNode(node *blockNode) (*blockNode, error) { + bi.Lock() + node, err := bi.prevNodeFromNode(node) + bi.Unlock() + return node, err +} + +// RelativeNode returns the ancestor block a relative 'distance' blocks before +// the passed anchor block. While iterating backwards through the chain, any +// block nodes which aren't in the memory chain are loaded in dynamically. +// +// This function is safe for concurrent access. +func (bi *blockIndex) RelativeNode(anchor *blockNode, distance uint32) (*blockNode, error) { + bi.Lock() + defer bi.Unlock() + + iterNode := anchor + err := bi.db.View(func(dbTx database.Tx) error { + // Walk backwards in the chian until we've gone 'distance' + // steps back. + var err error + for i := distance; i > 0; i-- { + switch { + // If the parent of this node has already been loaded + // into memory, then we can follow the link without + // hitting the database. + case iterNode.parent != nil: + iterNode = iterNode.parent + + // If this node is the genesis block, then we can't go + // back any further, so we exit immediately. + case iterNode.hash.IsEqual(bi.chainParams.GenesisHash): + return nil + + // Otherwise, load the block node from the database, + // pulling it into the memory cache in the processes. + default: + iterNode, err = bi.loadBlockNode(dbTx, + iterNode.parentHash) + if err != nil { + return err + } + } + } + + return nil + }) + if err != nil { + return nil, err + } + + return iterNode, nil +} + +// AncestorNode returns the ancestor block node at the provided height by +// following the chain backwards from the given node while dynamically loading +// any pruned nodes from the database and updating the memory block chain as +// needed. The returned block will be nil when a height is requested that is +// after the height of the passed node or is less than zero. +// +// This function is safe for concurrent access. +func (bi *blockIndex) AncestorNode(node *blockNode, height int32) (*blockNode, error) { + // Nothing to do if the requested height is outside of the valid range. + if height > node.height || height < 0 { + return nil, nil + } + + // Iterate backwards until the requested height is reached. + bi.Lock() + iterNode := node + for iterNode != nil && iterNode.height > height { + var err error + iterNode, err = bi.prevNodeFromNode(iterNode) + if err != nil { + break + } + } + bi.Unlock() + + return iterNode, nil +} + +// AddNode adds the provided node to the block index. Duplicate entries are not +// checked so it is up to caller to avoid adding them. +// +// This function is safe for concurrent access. +func (bi *blockIndex) AddNode(node *blockNode) { + bi.Lock() + bi.index[*node.hash] = node + if prevHash := node.parentHash; prevHash != nil && *prevHash != *zeroHash { + bi.depNodes[*prevHash] = append(bi.depNodes[*prevHash], node) + } + bi.Unlock() +} + +// LookupNode returns the block node identified by the provided hash. It will +// return nil if there is no entry for the hash. +// +// This function is safe for concurrent access. +func (bi *blockIndex) LookupNode(hash *chainhash.Hash) *blockNode { + bi.RLock() + node := bi.index[*hash] + bi.RUnlock() + return node +} + +// CalcPastMedianTime calculates the median time of the previous few blocks +// prior to, and including, the passed block node. +// +// This function is safe for concurrent access. +func (bi *blockIndex) CalcPastMedianTime(startNode *blockNode) (time.Time, error) { + // Genesis block. + if startNode == nil { + return bi.chainParams.GenesisBlock.Header.Timestamp, nil + } + + // Create a slice of the previous few block timestamps used to calculate + // the median per the number defined by the constant medianTimeBlocks. + timestamps := make([]int64, medianTimeBlocks) + numNodes := 0 + iterNode := startNode + bi.Lock() + for i := 0; i < medianTimeBlocks && iterNode != nil; i++ { + timestamps[i] = iterNode.timestamp + numNodes++ + + // Get the previous block node. This function is used over + // simply accessing iterNode.parent directly as it will + // dynamically create previous block nodes as needed. This + // helps allow only the pieces of the chain that are needed + // to remain in memory. + var err error + iterNode, err = bi.prevNodeFromNode(iterNode) + if err != nil { + bi.Unlock() + log.Errorf("prevNodeFromNode: %v", err) + return time.Time{}, err + } + } + bi.Unlock() + + // Prune the slice to the actual number of available timestamps which + // will be fewer than desired near the beginning of the block chain + // and sort them. + timestamps = timestamps[:numNodes] + sort.Sort(timeSorter(timestamps)) + + // NOTE: The consensus rules incorrectly calculate the median for even + // numbers of blocks. A true median averages the middle two elements + // for a set with an even number of elements in it. Since the constant + // for the previous number of blocks to be used is odd, this is only an + // issue for a few blocks near the beginning of the chain. I suspect + // this is an optimization even though the result is slightly wrong for + // a few of the first blocks since after the first few blocks, there + // will always be an odd number of blocks in the set per the constant. + // + // This code follows suit to ensure the same rules are used, however, be + // aware that should the medianTimeBlocks constant ever be changed to an + // even number, this code will be wrong. + medianTimestamp := timestamps[numNodes/2] + return time.Unix(medianTimestamp, 0), nil +} diff --git a/blockchain/blocklocator.go b/blockchain/blocklocator.go index 1182c9a3..612c3b27 100644 --- a/blockchain/blocklocator.go +++ b/blockchain/blocklocator.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2016 The btcsuite developers +// Copyright (c) 2013-2017 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -38,14 +38,14 @@ type BlockLocator []*chainhash.Hash // - If the passed hash is not currently known, the block locator will only // consist of the passed hash // -// This function MUST be called with the chain state lock held (for reads). -func (b *BlockChain) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { +// This function MUST be called with the block index lock held (for reads). +func (bi *blockIndex) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { // The locator contains the requested hash at the very least. locator := make(BlockLocator, 0, wire.MaxBlockLocatorsPerMsg) locator = append(locator, hash) // Nothing more to do if a locator for the genesis hash was requested. - if hash.IsEqual(b.chainParams.GenesisHash) { + if hash.IsEqual(bi.chainParams.GenesisHash) { return locator } @@ -54,13 +54,13 @@ func (b *BlockChain) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { // which it forks from the main chain. blockHeight := int32(-1) forkHeight := int32(-1) - node, exists := b.index[*hash] + node, exists := bi.index[*hash] if !exists { // Try to look up the height for passed block hash. Assume an // error means it doesn't exist and just return the locator for // the block itself. var height int32 - err := b.db.View(func(dbTx database.Tx) error { + err := bi.db.View(func(dbTx database.Tx) error { var err error height, err = dbFetchHeightByHash(dbTx, hash) return err @@ -93,7 +93,7 @@ func (b *BlockChain) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { // could fail is if there is something wrong with the database which // will be caught in short order anyways and it's also safe to ignore // block locators. - _ = b.db.View(func(dbTx database.Tx) error { + _ = bi.db.View(func(dbTx database.Tx) error { iterNode := node increment := int32(1) for len(locator) < wire.MaxBlockLocatorsPerMsg-1 { @@ -112,7 +112,7 @@ func (b *BlockChain) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { // height. if forkHeight != -1 && blockHeight > forkHeight { // Intentionally use parent field instead of the - // getPrevNodeFromNode function since we don't + // PrevNodeFromNode function since we don't // want to dynamically load nodes when building // block locators. Side chain blocks should // always be in memory already, and if they @@ -144,7 +144,7 @@ func (b *BlockChain) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { }) // Append the appropriate genesis block. - locator = append(locator, b.chainParams.GenesisHash) + locator = append(locator, bi.chainParams.GenesisHash) return locator } @@ -162,7 +162,9 @@ func (b *BlockChain) blockLocatorFromHash(hash *chainhash.Hash) BlockLocator { // This function is safe for concurrent access. func (b *BlockChain) BlockLocatorFromHash(hash *chainhash.Hash) BlockLocator { b.chainLock.RLock() - locator := b.blockLocatorFromHash(hash) + b.index.RLock() + locator := b.index.blockLocatorFromHash(hash) + b.index.RUnlock() b.chainLock.RUnlock() return locator } @@ -173,7 +175,9 @@ func (b *BlockChain) BlockLocatorFromHash(hash *chainhash.Hash) BlockLocator { // This function is safe for concurrent access. func (b *BlockChain) LatestBlockLocator() (BlockLocator, error) { b.chainLock.RLock() - locator := b.blockLocatorFromHash(b.bestNode.hash) + b.index.RLock() + locator := b.index.blockLocatorFromHash(b.bestNode.hash) + b.index.RUnlock() b.chainLock.RUnlock() return locator, nil } diff --git a/blockchain/chain.go b/blockchain/chain.go index 264a9f6a..e0e947fc 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -7,8 +7,6 @@ package blockchain import ( "container/list" "fmt" - "math/big" - "sort" "sync" "time" @@ -26,88 +24,6 @@ const ( maxOrphanBlocks = 100 ) -// blockNode represents a block within the block chain and is primarily used to -// aid in selecting the best chain to be the main chain. The main chain is -// stored into the block database. -type blockNode struct { - // parent is the parent block for this node. - parent *blockNode - - // children contains the child nodes for this node. Typically there - // will only be one, but sometimes there can be more than one and that - // is when the best chain selection algorithm is used. - children []*blockNode - - // hash is the double sha 256 of the block. - hash *chainhash.Hash - - // parentHash is the double sha 256 of the parent block. This is kept - // here over simply relying on parent.hash directly since block nodes - // are sparse and the parent node might not be in memory when its hash - // is needed. - parentHash *chainhash.Hash - - // height is the position in the block chain. - height int32 - - // workSum is the total amount of work in the chain up to and including - // this node. - workSum *big.Int - - // inMainChain denotes whether the block node is currently on the - // the main chain or not. This is used to help find the common - // ancestor when switching chains. - inMainChain bool - - // Some fields from block headers to aid in best chain selection and - // reconstructing headers from memory. These must be treated as - // immutable and are intentionally ordered to avoid padding on 64-bit - // platforms. - version int32 - bits uint32 - nonce uint32 - timestamp int64 - merkleRoot chainhash.Hash -} - -// newBlockNode returns a new block node for the given block header. It is -// completely disconnected from the chain and the workSum value is just the work -// for the passed block. The work sum is updated accordingly when the node is -// inserted into a chain. -func newBlockNode(blockHeader *wire.BlockHeader, blockHash *chainhash.Hash, height int32) *blockNode { - // Make a copy of the hash so the node doesn't keep a reference to part - // of the full block/block header preventing it from being garbage - // collected. - prevHash := blockHeader.PrevBlock - node := blockNode{ - hash: blockHash, - parentHash: &prevHash, - workSum: CalcWork(blockHeader.Bits), - height: height, - version: blockHeader.Version, - bits: blockHeader.Bits, - nonce: blockHeader.Nonce, - timestamp: blockHeader.Timestamp.Unix(), - merkleRoot: blockHeader.MerkleRoot, - } - return &node -} - -// Header constructs a block header from the node and returns it. -// -// This function is safe for concurrent access. -func (node *blockNode) Header() wire.BlockHeader { - // No lock is needed because all accessed fields are immutable. - return wire.BlockHeader{ - Version: node.version, - PrevBlock: *node.parentHash, - MerkleRoot: node.merkleRoot, - Timestamp: time.Unix(node.timestamp, 0), - Bits: node.bits, - Nonce: node.nonce, - } -} - // orphanBlock represents a block that we don't yet have the parent for. It // is a normal block plus an expiration time to prevent caching the orphan // forever. @@ -116,30 +32,6 @@ type orphanBlock struct { expiration time.Time } -// removeChildNode deletes node from the provided slice of child block -// nodes. It ensures the final pointer reference is set to nil to prevent -// potential memory leaks. The original slice is returned unmodified if node -// is invalid or not in the slice. -// -// This function MUST be called with the chain state lock held (for writes). -func removeChildNode(children []*blockNode, node *blockNode) []*blockNode { - if node == nil { - return children - } - - // An indexing for loop is intentionally used over a range here as range - // does not reevaluate the slice on each iteration nor does it adjust - // the index for the modified slice. - for i := 0; i < len(children); i++ { - if children[i].hash.IsEqual(node.hash) { - copy(children[i:], children[i+1:]) - children[len(children)-1] = nil - return children[:len(children)-1] - } - } - return children -} - // BestState houses information about the current best block and other info // related to the state of the main chain as it exists from the point of view of // the current best block. @@ -156,7 +48,7 @@ type BestState struct { BlockSize uint64 // The size of the block. NumTxns uint64 // The number of txns in the block. TotalTxns uint64 // The total number of txns in the chain. - MedianTime time.Time // Median time as per calcPastMedianTime. + MedianTime time.Time // Median time as per CalcPastMedianTime. } // newBestState returns a new best stats instance for the given parameters. @@ -213,11 +105,10 @@ type BlockChain struct { // runtime. They are protected by the chain lock. noVerify bool - // These fields are related to the memory block index. They are - // protected by the chain lock. + // These fields are related to the memory block index. The best node + // is protected by the chain lock and the index has its own locks. bestNode *blockNode - index map[chainhash.Hash]*blockNode - depNodes map[chainhash.Hash][]*blockNode + index *blockIndex // These fields are related to handling of orphan blocks. They are // protected by a combination of the chain lock and the orphan lock. @@ -434,264 +325,6 @@ func (b *BlockChain) addOrphanBlock(block *btcutil.Block) { return } -// loadBlockNode loads the block identified by hash from the block database, -// creates a block node from it, and updates the memory block chain accordingly. -// It is used mainly to dynamically load previous blocks from the database as -// they are needed to avoid needing to put the entire block chain in memory. -// -// This function MUST be called with the chain state lock held (for writes). -// The database transaction may be read-only. -func (b *BlockChain) loadBlockNode(dbTx database.Tx, hash *chainhash.Hash) (*blockNode, error) { - // Load the block header and height from the db. - blockHeader, err := dbFetchHeaderByHash(dbTx, hash) - if err != nil { - return nil, err - } - blockHeight, err := dbFetchHeightByHash(dbTx, hash) - if err != nil { - return nil, err - } - - // Create the new block node for the block and set the work. - node := newBlockNode(blockHeader, hash, blockHeight) - node.inMainChain = true - - // Add the node to the chain. - // There are a few possibilities here: - // 1) This node is a child of an existing block node - // 2) This node is the parent of one or more nodes - // 3) Neither 1 or 2 is true which implies it's an orphan block and - // therefore is an error to insert into the chain - prevHash := &blockHeader.PrevBlock - if parentNode, ok := b.index[*prevHash]; ok { - // Case 1 -- This node is a child of an existing block node. - // Update the node's work sum with the sum of the parent node's - // work sum and this node's work, append the node as a child of - // the parent node and set this node's parent to the parent - // node. - node.workSum = node.workSum.Add(parentNode.workSum, node.workSum) - parentNode.children = append(parentNode.children, node) - node.parent = parentNode - - } else if childNodes, ok := b.depNodes[*hash]; ok { - // Case 2 -- This node is the parent of one or more nodes. - // Update the node's work sum by subtracting this node's work - // from the sum of its first child, and connect the node to all - // of its children. - node.workSum.Sub(childNodes[0].workSum, node.workSum) - for _, childNode := range childNodes { - childNode.parent = node - node.children = append(node.children, childNode) - } - - } else { - // Case 3 -- The node doesn't have a parent and is not the - // parent of another node. This means an arbitrary orphan block - // is trying to be loaded which is not allowed. - str := "loadBlockNode: attempt to insert orphan block %v" - return nil, AssertError(fmt.Sprintf(str, hash)) - } - - // Add the new node to the indices for faster lookups. - b.index[*hash] = node - b.depNodes[*prevHash] = append(b.depNodes[*prevHash], node) - - return node, nil -} - -// getPrevNodeFromBlock returns a block node for the block previous to the -// passed block (the passed block's parent). When it is already in the memory -// block chain, it simply returns it. Otherwise, it loads the previous block -// header from the block database, creates a new block node from it, and returns -// it. The returned node will be nil if the genesis block is passed. -// -// This function MUST be called with the chain state lock held (for writes). -func (b *BlockChain) getPrevNodeFromBlock(block *btcutil.Block) (*blockNode, error) { - // Genesis block. - prevHash := &block.MsgBlock().Header.PrevBlock - if prevHash.IsEqual(zeroHash) { - return nil, nil - } - - // Return the existing previous block node if it's already there. - if bn, ok := b.index[*prevHash]; ok { - return bn, nil - } - - // Dynamically load the previous block from the block database, create - // a new block node for it, and update the memory chain accordingly. - var prevBlockNode *blockNode - err := b.db.View(func(dbTx database.Tx) error { - var err error - prevBlockNode, err = b.loadBlockNode(dbTx, prevHash) - return err - }) - return prevBlockNode, err -} - -// getPrevNodeFromNode returns a block node for the block previous to the -// passed block node (the passed block node's parent). When the node is already -// connected to a parent, it simply returns it. Otherwise, it loads the -// associated block from the database to obtain the previous hash and uses that -// to dynamically create a new block node and return it. The memory block -// chain is updated accordingly. The returned node will be nil if the genesis -// block is passed. -// -// This function MUST be called with the chain state lock held (for writes). -func (b *BlockChain) getPrevNodeFromNode(node *blockNode) (*blockNode, error) { - // Return the existing previous block node if it's already there. - if node.parent != nil { - return node.parent, nil - } - - // Genesis block. - if node.hash.IsEqual(b.chainParams.GenesisHash) { - return nil, nil - } - - // Dynamically load the previous block from the block database, create - // a new block node for it, and update the memory chain accordingly. - var prevBlockNode *blockNode - err := b.db.View(func(dbTx database.Tx) error { - var err error - prevBlockNode, err = b.loadBlockNode(dbTx, node.parentHash) - return err - }) - return prevBlockNode, err -} - -// relativeNode returns the ancestor block a relative 'distance' blocks before -// the passed anchor block. While iterating backwards through the chain, any -// block nodes which aren't in the memory chain are loaded in dynamically. -// -// This function MUST be called with the chain state lock held (for writes). -func (b *BlockChain) relativeNode(anchor *blockNode, distance uint32) (*blockNode, error) { - var err error - iterNode := anchor - - err = b.db.View(func(dbTx database.Tx) error { - // Walk backwards in the chian until we've gone 'distance' - // steps back. - for i := distance; i > 0; i-- { - switch { - // If the parent of this node has already been loaded - // into memory, then we can follow the link without - // hitting the database. - case iterNode.parent != nil: - iterNode = iterNode.parent - - // If this node is the genesis block, then we can't go - // back any further, so we exit immediately. - case iterNode.hash.IsEqual(b.chainParams.GenesisHash): - return nil - - // Otherwise, load the block node from the database, - // pulling it into the memory cache in the processes. - default: - iterNode, err = b.loadBlockNode(dbTx, - iterNode.parentHash) - if err != nil { - return err - } - } - } - - return nil - }) - if err != nil { - return nil, err - } - - return iterNode, nil -} - -// ancestorNode returns the ancestor block node at the provided height by -// following the chain backwards from the given node while dynamically loading -// any pruned nodes from the database and updating the memory block chain as -// needed. The returned block will be nil when a height is requested that is -// after the height of the passed node or is less than zero. -// -// This function MUST be called with the chain state lock held (for writes). -func (b *BlockChain) ancestorNode(node *blockNode, height int32) (*blockNode, error) { - // Nothing to do if the requested height is outside of the valid range. - if height > node.height || height < 0 { - return nil, nil - } - - // Iterate backwards until the requested height is reached. - iterNode := node - for iterNode != nil && iterNode.height > height { - // Get the previous block node. This function is used over - // simply accessing iterNode.parent directly as it will - // dynamically create previous block nodes as needed. This - // helps allow only the pieces of the chain that are needed - // to remain in memory. - var err error - iterNode, err = b.getPrevNodeFromNode(iterNode) - if err != nil { - log.Errorf("getPrevNodeFromNode: %v", err) - return nil, err - } - } - - return iterNode, nil -} - -// calcPastMedianTime calculates the median time of the previous few blocks -// prior to, and including, the passed block node. It is primarily used to -// validate new blocks have sane timestamps. -// -// This function MUST be called with the chain state lock held (for writes). -func (b *BlockChain) calcPastMedianTime(startNode *blockNode) (time.Time, error) { - // Genesis block. - if startNode == nil { - return b.chainParams.GenesisBlock.Header.Timestamp, nil - } - - // Create a slice of the previous few block timestamps used to calculate - // the median per the number defined by the constant medianTimeBlocks. - timestamps := make([]int64, medianTimeBlocks) - numNodes := 0 - iterNode := startNode - for i := 0; i < medianTimeBlocks && iterNode != nil; i++ { - timestamps[i] = iterNode.timestamp - numNodes++ - - // Get the previous block node. This function is used over - // simply accessing iterNode.parent directly as it will - // dynamically create previous block nodes as needed. This - // helps allow only the pieces of the chain that are needed - // to remain in memory. - var err error - iterNode, err = b.getPrevNodeFromNode(iterNode) - if err != nil { - log.Errorf("getPrevNodeFromNode: %v", err) - return time.Time{}, err - } - } - - // Prune the slice to the actual number of available timestamps which - // will be fewer than desired near the beginning of the block chain - // and sort them. - timestamps = timestamps[:numNodes] - sort.Sort(timeSorter(timestamps)) - - // NOTE: bitcoind incorrectly calculates the median for even numbers of - // blocks. A true median averages the middle two elements for a set - // with an even number of elements in it. Since the constant for the - // previous number of blocks to be used is odd, this is only an issue - // for a few blocks near the beginning of the chain. I suspect this is - // an optimization even though the result is slightly wrong for a few - // of the first blocks since after the first few blocks, there will - // always be an odd number of blocks in the set per the constant. - // - // This code follows suit to ensure the same rules are used as bitcoind - // however, be aware that should the medianTimeBlocks constant ever be - // changed to an even number, this code will be wrong. - medianTimestamp := timestamps[numNodes/2] - return time.Unix(medianTimestamp, 0), nil -} - // SequenceLock represents the converted relative lock-time in seconds, and // absolute block-height for a transaction input's relative lock-times. // According to SequenceLock, after the referenced input has been confirmed @@ -789,7 +422,8 @@ func (b *BlockChain) calcSequenceLock(tx *btcutil.Tx, utxoView *UtxoViewpoint, // the one which included this referenced output. // TODO: caching should be added to keep this speedy inputDepth := uint32(b.bestNode.height-inputHeight) + 1 - blockNode, err := b.relativeNode(b.bestNode, inputDepth) + blockNode, err := b.index.RelativeNode(b.bestNode, + inputDepth) if err != nil { return sequenceLock, err } @@ -798,7 +432,7 @@ func (b *BlockChain) calcSequenceLock(tx *btcutil.Tx, utxoView *UtxoViewpoint, // memory, we can now finally calculate the MTP of the // block prior to the one which included the output // being spent. - medianTime, err := b.calcPastMedianTime(blockNode) + medianTime, err := b.index.CalcPastMedianTime(blockNode) if err != nil { return sequenceLock, err } @@ -953,7 +587,7 @@ func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block, view *U } // Calculate the median time for the block. - medianTime, err := b.calcPastMedianTime(node) + medianTime, err := b.index.CalcPastMedianTime(node) if err != nil { return err } @@ -1023,11 +657,9 @@ func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block, view *U // now that the modifications have been committed to the database. view.commit() - // Add the new node to the memory main chain indices for faster - // lookups. + // Add the new node to the memory main chain indices for faster lookups. node.inMainChain = true - b.index[*node.hash] = node - b.depNodes[*prevHash] = append(b.depNodes[*prevHash], node) + b.index.AddNode(node) // This node is now the end of the best chain. b.bestNode = node @@ -1066,13 +698,13 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block *btcutil.Block, view // accessing node.parent directly as it will dynamically create previous // block nodes as needed. This helps allow only the pieces of the chain // that are needed to remain in memory. - prevNode, err := b.getPrevNodeFromNode(node) + prevNode, err := b.index.PrevNodeFromNode(node) if err != nil { return err } // Calculate the median time for the previous block. - medianTime, err := b.calcPastMedianTime(prevNode) + medianTime, err := b.index.CalcPastMedianTime(prevNode) if err != nil { return err } @@ -1371,7 +1003,7 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags // Log the point where the chain forked. firstAttachNode := attachNodes.Front().Value.(*blockNode) - forkNode, err := b.getPrevNodeFromNode(firstAttachNode) + forkNode, err := b.index.PrevNodeFromNode(firstAttachNode) if err == nil { log.Infof("REORGANIZE: Chain forks at %v", forkNode.hash) } @@ -1463,7 +1095,9 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla // We're extending (or creating) a side chain which may or may not // become the main chain, but in either case the entry is needed in the // index for future processing. - b.index[*node.hash] = node + b.index.Lock() + b.index.index[*node.hash] = node + b.index.Unlock() // Connect the parent node to this node. node.inMainChain = false @@ -1477,7 +1111,9 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla children = removeChildNode(children, node) node.parent.children = children - delete(b.index, *node.hash) + b.index.Lock() + delete(b.index.index, *node.hash) + b.index.Unlock() }() } @@ -1587,10 +1223,7 @@ func (b *BlockChain) BestSnapshot() *BestState { // if it doesn't exist. func (b *BlockChain) FetchHeader(hash *chainhash.Hash) (wire.BlockHeader, error) { // Reconstruct the header from the block index if possible. - b.chainLock.RLock() - node, ok := b.index[*hash] - b.chainLock.RUnlock() - if ok { + if node := b.index.LookupNode(hash); node != nil { return node.Header(), nil } @@ -1727,10 +1360,7 @@ func New(config *Config) (*BlockChain, error) { minRetargetTimespan: targetTimespan / adjustmentFactor, maxRetargetTimespan: targetTimespan * adjustmentFactor, blocksPerRetarget: int32(targetTimespan / targetTimePerBlock), - minMemoryNodes: int32(targetTimespan / targetTimePerBlock), - bestNode: nil, - index: make(map[chainhash.Hash]*blockNode), - depNodes: make(map[chainhash.Hash][]*blockNode), + index: newBlockIndex(config.DB, params), orphans: make(map[chainhash.Hash]*orphanBlock), prevOrphans: make(map[chainhash.Hash][]*orphanBlock), warningCaches: newThresholdCaches(vbNumBits), diff --git a/blockchain/chainio.go b/blockchain/chainio.go index cc9c0303..bd92a389 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1086,7 +1086,7 @@ func (b *BlockChain) createChainState() error { b.bestNode = node // Add the new node to the index which is used for faster lookups. - b.index[*node.hash] = node + b.index.AddNode(node) // Initialize the state related to the best block. Since it is the // genesis block, use its timestamp for the median time. @@ -1186,13 +1186,11 @@ func (b *BlockChain) initChainState() error { node.workSum = state.workSum b.bestNode = node - // Add the new node to the indices for faster lookups. - prevHash := node.parentHash - b.index[*node.hash] = node - b.depNodes[*prevHash] = append(b.depNodes[*prevHash], node) + // Add the new node to the block index. + b.index.AddNode(node) // Calculate the median time for the block. - medianTime, err := b.calcPastMedianTime(node) + medianTime, err := b.index.CalcPastMedianTime(node) if err != nil { return err } @@ -1910,7 +1908,7 @@ func (b *BlockChain) initThresholdCaches() error { // accessing b.bestNode.parent directly as it will dynamically create // previous block nodes as needed. This helps allow only the pieces of // the chain that are needed to remain in memory. - prevNode, err := b.getPrevNodeFromNode(b.bestNode) + prevNode, err := b.index.PrevNodeFromNode(b.bestNode) if err != nil { return err } diff --git a/blockchain/difficulty.go b/blockchain/difficulty.go index f6a0689e..755e0979 100644 --- a/blockchain/difficulty.go +++ b/blockchain/difficulty.go @@ -207,9 +207,9 @@ func (b *BlockChain) findPrevTestNetDifficulty(startNode *blockNode) (uint32, er // helps allow only the pieces of the chain that are needed // to remain in memory. var err error - iterNode, err = b.getPrevNodeFromNode(iterNode) + iterNode, err = b.index.PrevNodeFromNode(iterNode) if err != nil { - log.Errorf("getPrevNodeFromNode: %v", err) + log.Errorf("PrevNodeFromNode: %v", err) return 0, err } } @@ -277,7 +277,7 @@ func (b *BlockChain) calcNextRequiredDifficulty(lastNode *blockNode, newBlockTim // helps allow only the pieces of the chain that are needed // to remain in memory. var err error - firstNode, err = b.getPrevNodeFromNode(firstNode) + firstNode, err = b.index.PrevNodeFromNode(firstNode) if err != nil { return 0, err } diff --git a/blockchain/process.go b/blockchain/process.go index 143f555c..5a2e5ab1 100644 --- a/blockchain/process.go +++ b/blockchain/process.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2016 The btcsuite developers +// Copyright (c) 2013-2017 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -42,8 +42,8 @@ const ( // // This function MUST be called with the chain state lock held (for reads). func (b *BlockChain) blockExists(hash *chainhash.Hash) (bool, error) { - // Check memory chain first (could be main chain or side chain blocks). - if _, ok := b.index[*hash]; ok { + // Check block index first (could be main chain or side chain blocks). + if b.index.HaveBlock(hash) { return true, nil } diff --git a/blockchain/thresholdstate.go b/blockchain/thresholdstate.go index f1ca8d19..a4953bbd 100644 --- a/blockchain/thresholdstate.go +++ b/blockchain/thresholdstate.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The btcsuite developers +// Copyright (c) 2016-2017 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -158,7 +158,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit // window in order to get its threshold state. This can be done because // the state is the same for all blocks within a given window. var err error - prevNode, err = b.ancestorNode(prevNode, prevNode.height- + prevNode, err = b.index.AncestorNode(prevNode, prevNode.height- (prevNode.height+1)%confirmationWindow) if err != nil { return ThresholdFailed, err @@ -176,7 +176,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit // The start and expiration times are based on the median block // time, so calculate it now. - medianTime, err := b.calcPastMedianTime(prevNode) + medianTime, err := b.index.CalcPastMedianTime(prevNode) if err != nil { return ThresholdFailed, err } @@ -194,7 +194,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit // Get the ancestor that is the last block of the previous // confirmation window. - prevNode, err = b.ancestorNode(prevNode, prevNode.height- + prevNode, err = b.index.AncestorNode(prevNode, prevNode.height- confirmationWindow) if err != nil { return ThresholdFailed, err @@ -223,7 +223,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit case ThresholdDefined: // The deployment of the rule change fails if it expires // before it is accepted and locked in. - medianTime, err := b.calcPastMedianTime(prevNode) + medianTime, err := b.index.CalcPastMedianTime(prevNode) if err != nil { return ThresholdFailed, err } @@ -243,7 +243,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit case ThresholdStarted: // The deployment of the rule change fails if it expires // before it is accepted and locked in. - medianTime, err := b.calcPastMedianTime(prevNode) + medianTime, err := b.index.CalcPastMedianTime(prevNode) if err != nil { return ThresholdFailed, err } @@ -272,7 +272,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit // previous block nodes as needed. This helps // allow only the pieces of the chain that are // needed to remain in memory. - countNode, err = b.getPrevNodeFromNode(countNode) + countNode, err = b.index.PrevNodeFromNode(countNode) if err != nil { return ThresholdFailed, err } @@ -305,7 +305,7 @@ func (b *BlockChain) thresholdState(prevNode *blockNode, checker thresholdCondit } // ThresholdState returns the current rule change threshold state of the given -// deployment ID for the block AFTER then end of the current best chain. +// deployment ID for the block AFTER the end of the current best chain. // // This function is safe for concurrent access. func (b *BlockChain) ThresholdState(deploymentID uint32) (ThresholdState, error) { diff --git a/blockchain/validate.go b/blockchain/validate.go index 822c1791..668036cb 100644 --- a/blockchain/validate.go +++ b/blockchain/validate.go @@ -671,9 +671,9 @@ func (b *BlockChain) checkBlockHeaderContext(header *wire.BlockHeader, prevNode // Ensure the timestamp for the block header is after the // median time of the last several blocks (medianTimeBlocks). - medianTime, err := b.calcPastMedianTime(prevNode) + medianTime, err := b.index.CalcPastMedianTime(prevNode) if err != nil { - log.Errorf("calcPastMedianTime: %v", err) + log.Errorf("CalcPastMedianTime: %v", err) return err } if !header.Timestamp.After(medianTime) { diff --git a/blockchain/versionbits.go b/blockchain/versionbits.go index 4da737d9..cbf9e40c 100644 --- a/blockchain/versionbits.go +++ b/blockchain/versionbits.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The btcsuite developers +// Copyright (c) 2016-2017 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -117,7 +117,7 @@ func (c bitConditionChecker) Condition(node *blockNode) (bool, error) { // accessing node.parent directly as it will dynamically create previous // block nodes as needed. This helps allow only the pieces of the chain // that are needed to remain in memory. - prevNode, err := c.chain.getPrevNodeFromNode(node) + prevNode, err := c.chain.index.PrevNodeFromNode(node) if err != nil { return false, err } @@ -248,7 +248,7 @@ func (b *BlockChain) warnUnknownRuleActivations(node *blockNode) error { // accessing node.parent directly as it will dynamically create previous // block nodes as needed. This helps allow only the pieces of the chain // that are needed to remain in memory. - prevNode, err := b.getPrevNodeFromNode(node) + prevNode, err := b.index.PrevNodeFromNode(node) if err != nil { return err } @@ -309,7 +309,7 @@ func (b *BlockChain) warnUnknownVersions(node *blockNode) error { // simply accessing node.parent directly as it will dynamically // create previous block nodes as needed. This helps allow only // the pieces of the chain that are needed to remain in memory. - node, err = b.getPrevNodeFromNode(node) + node, err = b.index.PrevNodeFromNode(node) if err != nil { return err }