From 491acd4ca6f88d18afdb0294e3899257140c0df0 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 25 Aug 2015 23:03:18 -0500 Subject: [PATCH] blockchain: Rework to use new db interface. This commit is the first stage of several that are planned to convert the blockchain package into a concurrent safe package that will ultimately allow support for multi-peer download and concurrent chain processing. The goal is to update btcd proper after each step so it can take advantage of the enhancements as they are developed. In addition to the aforementioned benefit, this staged approach has been chosen since it is absolutely critical to maintain consensus. Separating the changes into several stages makes it easier for reviewers to logically follow what is happening and therefore helps prevent consensus bugs. Naturally there are significant automated tests to help prevent consensus issues as well. The main focus of this stage is to convert the blockchain package to use the new database interface and implement the chain-related functionality which it no longer handles. It also aims to improve efficiency in various areas by making use of the new database and chain capabilities. The following is an overview of the chain changes: - Update to use the new database interface - Add chain-related functionality that the old database used to handle - Main chain structure and state - Transaction spend tracking - Implement a new pruned unspent transaction output (utxo) set - Provides efficient direct access to the unspent transaction outputs - Uses a domain specific compression algorithm that understands the standard transaction scripts in order to significantly compress them - Removes reliance on the transaction index and paves the way toward eventually enabling block pruning - Modify the New function to accept a Config struct instead of inidividual parameters - Replace the old TxStore type with a new UtxoViewpoint type that makes use of the new pruned utxo set - Convert code to treat the new UtxoViewpoint as a rolling view that is used between connects and disconnects to improve efficiency - Make best chain state always set when the chain instance is created - Remove now unnecessary logic for dealing with unset best state - Make all exported functions concurrent safe - Currently using a single chain state lock as it provides a straight forward and easy to review path forward however this can be improved with more fine grained locking - Optimize various cases where full blocks were being loaded when only the header is needed to help reduce the I/O load - Add the ability for callers to get a snapshot of the current best chain stats in a concurrent safe fashion - Does not block callers while new blocks are being processed - Make error messages that reference transaction outputs consistently use : - Introduce a new AssertError type an convert internal consistency checks to use it - Update tests and examples to reflect the changes - Add a full suite of tests to ensure correct functionality of the new code The following is an overview of the btcd changes: - Update to use the new database and chain interfaces - Temporarily remove all code related to the transaction index - Temporarily remove all code related to the address index - Convert all code that uses transaction stores to use the new utxo view - Rework several calls that required the block manager for safe concurrency to use the chain package directly now that it is concurrent safe - Change all calls to obtain the best hash to use the new best state snapshot capability from the chain package - Remove workaround for limits on fetching height ranges since the new database interface no longer imposes them - Correct the gettxout RPC handler to return the best chain hash as opposed the hash the txout was found in - Optimize various RPC handlers: - Change several of the RPC handlers to use the new chain snapshot capability to avoid needlessly loading data - Update several handlers to use new functionality to avoid accessing the block manager so they are able to return the data without blocking when the server is busy processing blocks - Update non-verbose getblock to avoid deserialization and serialization overhead - Update getblockheader to request the block height directly from chain and only load the header - Update getdifficulty to use the new cached data from chain - Update getmininginfo to use the new cached data from chain - Update non-verbose getrawtransaction to avoid deserialization and serialization overhead - Update gettxout to use the new utxo store versus loading full transactions using the transaction index The following is an overview of the utility changes: - Update addblock to use the new database and chain interfaces - Update findcheckpoint to use the new database and chain interfaces - Remove the dropafter utility which is no longer supported NOTE: The transaction index and address index will be reimplemented in another commit. --- blockchain/accept.go | 6 +- blockchain/blocklocator.go | 153 ++- blockchain/chain.go | 767 ++++++++---- blockchain/chainio.go | 1401 ++++++++++++++++++++++ blockchain/chainio_test.go | 1030 ++++++++++++++++ blockchain/checkpoints.go | 218 ++-- blockchain/common_test.go | 164 +-- blockchain/compress.go | 603 ++++++++++ blockchain/compress_test.go | 491 ++++++++ blockchain/difficulty.go | 16 +- blockchain/error.go | 12 +- blockchain/example_test.go | 41 +- blockchain/internal_test.go | 6 +- blockchain/notifications.go | 2 +- blockchain/process.go | 22 +- blockchain/scriptval.go | 45 +- blockchain/scriptval_test.go | 12 +- blockchain/testdata/277647.txstore.bz2 | Bin 1374217 -> 0 bytes blockchain/testdata/277647.utxostore.bz2 | Bin 0 -> 36632 bytes blockchain/txlookup.go | 318 ----- blockchain/utxoviewpoint.go | 611 ++++++++++ blockchain/validate.go | 339 +++--- blockchain/validate_test.go | 12 +- blockmanager.go | 302 ++--- btcd.go | 15 +- chainindexer.go | 492 -------- cmd/addblock/addblock.go | 33 +- cmd/addblock/config.go | 10 +- cmd/addblock/import.go | 25 +- cmd/dropafter/dropafter.go | 227 ---- cmd/findcheckpoint/config.go | 10 +- cmd/findcheckpoint/findcheckpoint.go | 44 +- config.go | 31 +- doc.go | 9 +- log.go | 2 +- mempool.go | 261 +--- mining.go | 123 +- policy.go | 40 +- rpcserver.go | 975 +++++---------- rpcwebsocket.go | 73 +- sample-btcd.conf | 5 + server.go | 234 ++-- wire/msgheaders.go | 3 +- 43 files changed, 5960 insertions(+), 3223 deletions(-) create mode 100644 blockchain/chainio.go create mode 100644 blockchain/chainio_test.go create mode 100644 blockchain/compress.go create mode 100644 blockchain/compress_test.go delete mode 100644 blockchain/testdata/277647.txstore.bz2 create mode 100644 blockchain/testdata/277647.utxostore.bz2 delete mode 100644 blockchain/txlookup.go create mode 100644 blockchain/utxoviewpoint.go delete mode 100644 chainindexer.go delete mode 100644 cmd/dropafter/dropafter.go diff --git a/blockchain/accept.go b/blockchain/accept.go index f5dd2842..938d40a8 100644 --- a/blockchain/accept.go +++ b/blockchain/accept.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2015 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -17,6 +17,8 @@ import "github.com/btcsuite/btcutil" // // The flags are also passed to checkBlockContext and connectBestChain. See // their documentation for how the flags modify their behavior. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) error { dryRun := flags&BFDryRun == BFDryRun @@ -74,7 +76,9 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) // chain. The caller would typically want to react by relaying the // inventory to other peers. if !dryRun { + b.chainLock.Unlock() b.sendNotification(NTBlockAccepted, block) + b.chainLock.Lock() } return nil diff --git a/blockchain/blocklocator.go b/blockchain/blocklocator.go index c05de0cf..741813fd 100644 --- a/blockchain/blocklocator.go +++ b/blockchain/blocklocator.go @@ -1,10 +1,11 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package blockchain import ( + database "github.com/btcsuite/btcd/database2" "github.com/btcsuite/btcd/wire" ) @@ -25,7 +26,7 @@ import ( // [17a 16a 15 14 13 12 11 10 9 8 6 2 genesis] type BlockLocator []*wire.ShaHash -// BlockLocatorFromHash returns a block locator for the passed block hash. +// blockLocatorFromHash returns a block locator for the passed block hash. // See BlockLocator for details on the algotirhm used to create a block locator. // // In addition to the general algorithm referenced above, there are a couple of @@ -35,7 +36,9 @@ type BlockLocator []*wire.ShaHash // therefore the block locator will only consist of the genesis hash // - If the passed hash is not currently known, the block locator will only // consist of the passed hash -func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { +// +// This function MUST be called with the chain state lock held (for reads). +func (b *BlockChain) blockLocatorFromHash(hash *wire.ShaHash) BlockLocator { // The locator contains the requested hash at the very least. locator := make(BlockLocator, 0, wire.MaxBlockLocatorsPerMsg) locator = append(locator, hash) @@ -55,7 +58,12 @@ func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { // Try to look up the height for passed block hash. Assume an // error means it doesn't exist and just return the locator for // the block itself. - height, err := b.db.FetchBlockHeightBySha(hash) + var height int32 + err := b.db.View(func(dbTx database.Tx) error { + var err error + height, err = dbFetchHeightByHash(dbTx, hash) + return err + }) if err != nil { return locator } @@ -77,73 +85,94 @@ func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { } // Generate the block locators according to the algorithm described in - // in the BlockLocator comment and make sure to leave room for the - // final genesis hash. - iterNode := node - increment := int32(1) - for len(locator) < wire.MaxBlockLocatorsPerMsg-1 { - // Once there are 10 locators, exponentially increase the - // distance between each block locator. - if len(locator) > 10 { - increment *= 2 - } - blockHeight -= increment - if blockHeight < 1 { - break + // in the BlockLocator comment and make sure to leave room for the final + // genesis hash. + // + // The error is intentionally ignored here since the only way the code + // could fail is if there is something wrong with the database which + // will be caught in short order anyways and it's also safe to ignore + // block locators. + _ = b.db.View(func(dbTx database.Tx) error { + iterNode := node + increment := int32(1) + for len(locator) < wire.MaxBlockLocatorsPerMsg-1 { + // Once there are 10 locators, exponentially increase + // the distance between each block locator. + if len(locator) > 10 { + increment *= 2 + } + blockHeight -= increment + if blockHeight < 1 { + break + } + + // As long as this is still on the side chain, walk + // backwards along the side chain nodes to each block + // height. + if forkHeight != -1 && blockHeight > forkHeight { + // Intentionally use parent field instead of the + // getPrevNodeFromNode function since we don't + // want to dynamically load nodes when building + // block locators. Side chain blocks should + // always be in memory already, and if they + // aren't for some reason it's ok to skip them. + for iterNode != nil && blockHeight > iterNode.height { + iterNode = iterNode.parent + } + if iterNode != nil && iterNode.height == blockHeight { + locator = append(locator, iterNode.hash) + } + continue + } + + // The desired block height is in the main chain, so + // look it up from the main chain database. + h, err := dbFetchHashByHeight(dbTx, blockHeight) + if err != nil { + // This shouldn't happen and it's ok to ignore + // block locators, so just continue to the next + // one. + log.Warnf("Lookup of known valid height failed %v", + blockHeight) + continue + } + locator = append(locator, h) } - // As long as this is still on the side chain, walk backwards - // along the side chain nodes to each block height. - if forkHeight != -1 && blockHeight > forkHeight { - // Intentionally use parent field instead of the - // getPrevNodeFromNode function since we don't want to - // dynamically load nodes when building block locators. - // Side chain blocks should always be in memory already, - // and if they aren't for some reason it's ok to skip - // them. - for iterNode != nil && blockHeight > iterNode.height { - iterNode = iterNode.parent - } - if iterNode != nil && iterNode.height == blockHeight { - locator = append(locator, iterNode.hash) - } - continue - } - - // The desired block height is in the main chain, so look it up - // from the main chain database. - h, err := b.db.FetchBlockShaByHeight(blockHeight) - if err != nil { - // This shouldn't happen and it's ok to ignore block - // locators, so just continue to the next one. - log.Warnf("Lookup of known valid height failed %v", - blockHeight) - continue - } - locator = append(locator, h) - } + return nil + }) // Append the appropriate genesis block. locator = append(locator, b.chainParams.GenesisHash) return locator } +// BlockLocatorFromHash returns a block locator for the passed block hash. +// See BlockLocator for details on the algorithm used to create a block locator. +// +// In addition to the general algorithm referenced above, there are a couple of +// special cases which are handled: +// +// - If the genesis hash is passed, there are no previous hashes to add and +// therefore the block locator will only consist of the genesis hash +// - If the passed hash is not currently known, the block locator will only +// consist of the passed hash +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { + b.chainLock.RLock() + locator := b.blockLocatorFromHash(hash) + b.chainLock.RUnlock() + return locator +} + // LatestBlockLocator returns a block locator for the latest known tip of the // main (best) chain. +// +// This function is safe for concurrent access. func (b *BlockChain) LatestBlockLocator() (BlockLocator, error) { - // Lookup the latest main chain hash if the best chain hasn't been set - // yet. - if b.bestChain == nil { - // Get the latest block hash for the main chain from the - // database. - hash, _, err := b.db.NewestSha() - if err != nil { - return nil, err - } - - return b.BlockLocatorFromHash(hash), nil - } - - // The best chain is set, so use its hash. - return b.BlockLocatorFromHash(b.bestChain.hash), nil + b.chainLock.RLock() + locator := b.blockLocatorFromHash(b.bestNode.hash) + b.chainLock.RUnlock() + return locator, nil } diff --git a/blockchain/chain.go b/blockchain/chain.go index b4775a59..1026c2e4 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -6,7 +6,6 @@ package blockchain import ( "container/list" - "errors" "fmt" "math/big" "sort" @@ -14,7 +13,7 @@ import ( "time" "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/database" + database "github.com/btcsuite/btcd/database2" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" @@ -32,11 +31,6 @@ const ( minMemoryNodes = BlocksPerRetarget ) -// ErrIndexAlreadyInitialized describes an error that indicates the block index -// is already initialized. -var ErrIndexAlreadyInitialized = errors.New("the block index can only be " + - "initialized before it has been modified") - // blockNode represents a block within the block chain and is primarily used to // aid in selecting the best chain to be the main chain. The main chain is // stored into the block database. @@ -105,21 +99,12 @@ type orphanBlock struct { expiration time.Time } -// addChildrenWork adds the passed work amount to all children all the way -// down the chain. It is used primarily to allow a new node to be dynamically -// inserted from the database into the memory chain prior to nodes we already -// have and update their work values accordingly. -func addChildrenWork(node *blockNode, work *big.Int) { - for _, childNode := range node.children { - childNode.workSum.Add(childNode.workSum, work) - addChildrenWork(childNode, work) - } -} - // removeChildNode deletes node from the provided slice of child block // nodes. It ensures the final pointer reference is set to nil to prevent // potential memory leaks. The original slice is returned unmodified if node // is invalid or not in the slice. +// +// This function MUST be called with the chain state lock held (for writes). func removeChildNode(children []*blockNode, node *blockNode) []*blockNode { if node == nil { return children @@ -138,46 +123,116 @@ func removeChildNode(children []*blockNode, node *blockNode) []*blockNode { return children } +// BestState houses information about the current best block and other info +// related to the state of the main chain as it exists from the point of view of +// the current best block. +// +// The BestSnapshot method can be used to obtain access to this information +// in a concurrent safe manner and the data will not be changed out from under +// the caller when chain state changes occur as the function name implies. +// However, the returned snapshot must be treated as immutable since it is +// shared by all callers. +type BestState struct { + Hash *wire.ShaHash // The hash of the block. + Height int32 // The height of the block. + Bits uint32 // The difficulty bits of the block. + BlockSize uint64 // The size of the block. + NumTxns uint64 // The number of txns in the block. + TotalTxns uint64 // The total number of txns in the chain. +} + +// newBestState returns a new best stats instance for the given parameters. +func newBestState(node *blockNode, blockSize, numTxns, totalTxns uint64) *BestState { + return &BestState{ + Hash: node.hash, + Height: node.height, + Bits: node.bits, + BlockSize: blockSize, + NumTxns: numTxns, + TotalTxns: totalTxns, + } +} + // BlockChain provides functions for working with the bitcoin block chain. // It includes functionality such as rejecting duplicate blocks, ensuring blocks // follow all rules, orphan handling, checkpoint handling, and best chain // selection with reorganization. type BlockChain struct { - db database.Db - chainParams *chaincfg.Params + // The following fields are set when the instance is created and can't + // be changed afterwards, so there is no need to protect them with a + // separate mutex. checkpointsByHeight map[int32]*chaincfg.Checkpoint + db database.DB + chainParams *chaincfg.Params notifications NotificationCallback - root *blockNode - bestChain *blockNode - index map[wire.ShaHash]*blockNode - depNodes map[wire.ShaHash][]*blockNode - orphans map[wire.ShaHash]*orphanBlock - prevOrphans map[wire.ShaHash][]*orphanBlock - oldestOrphan *orphanBlock - orphanLock sync.RWMutex - blockCache map[wire.ShaHash]*btcutil.Block - noVerify bool - noCheckpoints bool - nextCheckpoint *chaincfg.Checkpoint - checkpointBlock *btcutil.Block sigCache *txscript.SigCache + + // chainLock protects concurrent access to the vast majority of the + // fields in this struct below this point. + chainLock sync.RWMutex + + // These fields are configuration parameters that can be toggled at + // runtime. They are protected by the chain lock. + noVerify bool + noCheckpoints bool + + // These fields are related to the memory block index. They are + // protected by the chain lock. + root *blockNode + bestNode *blockNode + index map[wire.ShaHash]*blockNode + depNodes map[wire.ShaHash][]*blockNode + + // These fields are related to handling of orphan blocks. They are + // protected by a combination of the chain lock and the orphan lock. + orphanLock sync.RWMutex + orphans map[wire.ShaHash]*orphanBlock + prevOrphans map[wire.ShaHash][]*orphanBlock + oldestOrphan *orphanBlock + blockCache map[wire.ShaHash]*btcutil.Block + + // These fields are related to checkpoint handling. They are protected + // by the chain lock. + nextCheckpoint *chaincfg.Checkpoint + checkpointBlock *btcutil.Block + + // The state is used as a fairly efficient way to cache information + // about the current best chain state that is returned to callers when + // requested. It operates on the principle of MVCC such that any time a + // new block becomes the best block, the state pointer is replaced with + // a new struct and the old state is left untouched. In this way, + // multiple callers can be pointing to different best chain states. + // This is acceptable for most callers because the state is only being + // queried at a specific point in time. + // + // In addition, some of the fields are stored in the database so the + // chain state can be quickly reconstructed on load. + stateLock sync.RWMutex + stateSnapshot *BestState } // DisableVerify provides a mechanism to disable transaction script validation // which you DO NOT want to do in production as it could allow double spends -// and othe undesirable things. It is provided only for debug purposes since +// and other undesirable things. It is provided only for debug purposes since // script validation is extremely intensive and when debugging it is sometimes // nice to quickly get the chain. +// +// This function is safe for concurrent access. func (b *BlockChain) DisableVerify(disable bool) { + b.chainLock.Lock() b.noVerify = disable + b.chainLock.Unlock() } // HaveBlock returns whether or not the chain instance has the block represented // by the passed hash. This includes checking the various places a block can // be like part of the main chain, on a side chain, or in the orphan pool. // -// This function is NOT safe for concurrent access. +// This function is safe for concurrent access. func (b *BlockChain) HaveBlock(hash *wire.ShaHash) (bool, error) { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + exists, err := b.blockExists(hash) if err != nil { return false, err @@ -319,90 +374,20 @@ func (b *BlockChain) addOrphanBlock(block *btcutil.Block) { return } -// GenerateInitialIndex is an optional function which generates the required -// number of initial block nodes in an optimized fashion. This is optional -// because the memory block index is sparse and previous nodes are dynamically -// loaded as needed. However, during initial startup (when there are no nodes -// in memory yet), dynamically loading all of the required nodes on the fly in -// the usual way is much slower than preloading them. -// -// This function can only be called once and it must be called before any nodes -// are added to the block index. ErrIndexAlreadyInitialized is returned if -// the former is not the case. In practice, this means the function should be -// called directly after New. -func (b *BlockChain) GenerateInitialIndex() error { - // Return an error if the has already been modified. - if b.root != nil { - return ErrIndexAlreadyInitialized - } - - // Grab the latest block height for the main chain from the database. - _, endHeight, err := b.db.NewestSha() - if err != nil { - return err - } - - // Calculate the starting height based on the minimum number of nodes - // needed in memory. - startHeight := endHeight - minMemoryNodes - if startHeight < 0 { - startHeight = 0 - } - - // Loop forwards through each block loading the node into the index for - // the block. - // - // Due to a bug in the SQLite btcdb driver, the FetchBlockBySha call is - // limited to a maximum number of hashes per invocation. Since SQLite - // is going to be nuked eventually, the bug isn't being fixed in the - // driver. In the mean time, work around the issue by calling - // FetchBlockBySha multiple times with the appropriate indices as needed. - for start := startHeight; start <= endHeight; { - hashList, err := b.db.FetchHeightRange(start, endHeight+1) - if err != nil { - return err - } - - // The database did not return any further hashes. Break out of - // the loop now. - if len(hashList) == 0 { - break - } - - // Loop forwards through each block loading the node into the - // index for the block. - for _, hash := range hashList { - // Make a copy of the hash to make sure there are no - // references into the list so it can be freed. - hashCopy := hash - node, err := b.loadBlockNode(&hashCopy) - if err != nil { - return err - } - - // This node is now the end of the best chain. - b.bestChain = node - } - - // Start at the next block after the latest one on the next loop - // iteration. - start += int32(len(hashList)) - } - - return nil -} - // loadBlockNode loads the block identified by hash from the block database, // creates a block node from it, and updates the memory block chain accordingly. -// It is used mainly to dynamically load previous blocks from database as they -// are needed to avoid needing to put the entire block chain in memory. -func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { +// It is used mainly to dynamically load previous blocks from the database as +// they are needed to avoid needing to put the entire block chain in memory. +// +// This function MUST be called with the chain state lock held (for writes). +// The database transaction may be read-only. +func (b *BlockChain) loadBlockNode(dbTx database.Tx, hash *wire.ShaHash) (*blockNode, error) { // Load the block header and height from the db. - blockHeader, err := b.db.FetchBlockHeaderBySha(hash) + blockHeader, err := dbFetchHeaderByHash(dbTx, hash) if err != nil { return nil, err } - blockHeight, err := b.db.FetchBlockHeightBySha(hash) + blockHeight, err := dbFetchHeightByHash(dbTx, hash) if err != nil { return nil, err } @@ -412,14 +397,11 @@ func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { node.inMainChain = true // Add the node to the chain. - // There are several possibilities here: + // There are a few possibilities here: // 1) This node is a child of an existing block node // 2) This node is the parent of one or more nodes - // 3) Neither 1 or 2 is true, and this is not the first node being - // added to the tree which implies it's an orphan block and + // 3) Neither 1 or 2 is true which implies it's an orphan block and // therefore is an error to insert into the chain - // 4) Neither 1 or 2 is true, but this is the first node being added - // to the tree, so it's the root. prevHash := &blockHeader.PrevBlock if parentNode, ok := b.index[*prevHash]; ok { // Case 1 -- This node is a child of an existing block node. @@ -433,28 +415,22 @@ func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { } else if childNodes, ok := b.depNodes[*hash]; ok { // Case 2 -- This node is the parent of one or more nodes. - // Connect this block node to all of its children and update - // all of the children (and their children) with the new work - // sums. + // Update the node's work sum by subtracting this node's work + // from the sum of its first child, and connect the node to all + // of its children. + node.workSum.Sub(childNodes[0].workSum, node.workSum) for _, childNode := range childNodes { childNode.parent = node node.children = append(node.children, childNode) - addChildrenWork(childNode, node.workSum) b.root = node } } else { - // Case 3 -- The node does't have a parent and is not the parent - // of another node. This is only acceptable for the first node - // inserted into the chain. Otherwise it means an arbitrary - // orphan block is trying to be loaded which is not allowed. - if b.root != nil { - str := "loadBlockNode: attempt to insert orphan block %v" - return nil, fmt.Errorf(str, hash) - } - - // Case 4 -- This is the root since it's the first and only node. - b.root = node + // Case 3 -- The node doesn't have a parent and is not the + // parent of another node. This means an arbitrary orphan block + // is trying to be loaded which is not allowed. + str := "loadBlockNode: attempt to insert orphan block %v" + return nil, AssertError(fmt.Sprintf(str, hash)) } // Add the new node to the indices for faster lookups. @@ -467,8 +443,10 @@ func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { // getPrevNodeFromBlock returns a block node for the block previous to the // passed block (the passed block's parent). When it is already in the memory // block chain, it simply returns it. Otherwise, it loads the previous block -// from the block database, creates a new block node from it, and returns it. -// The returned node will be nil if the genesis block is passed. +// header from the block database, creates a new block node from it, and returns +// it. The returned node will be nil if the genesis block is passed. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) getPrevNodeFromBlock(block *btcutil.Block) (*blockNode, error) { // Genesis block. prevHash := &block.MsgBlock().Header.PrevBlock @@ -483,11 +461,13 @@ func (b *BlockChain) getPrevNodeFromBlock(block *btcutil.Block) (*blockNode, err // Dynamically load the previous block from the block database, create // a new block node for it, and update the memory chain accordingly. - prevBlockNode, err := b.loadBlockNode(prevHash) - if err != nil { - return nil, err - } - return prevBlockNode, nil + var prevBlockNode *blockNode + err := b.db.View(func(dbTx database.Tx) error { + var err error + prevBlockNode, err = b.loadBlockNode(dbTx, prevHash) + return err + }) + return prevBlockNode, err } // getPrevNodeFromNode returns a block node for the block previous to the @@ -497,6 +477,8 @@ func (b *BlockChain) getPrevNodeFromBlock(block *btcutil.Block) (*blockNode, err // to dynamically create a new block node and return it. The memory block // chain is updated accordingly. The returned node will be nil if the genesis // block is passed. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) getPrevNodeFromNode(node *blockNode) (*blockNode, error) { // Return the existing previous block node if it's already there. if node.parent != nil { @@ -510,21 +492,25 @@ func (b *BlockChain) getPrevNodeFromNode(node *blockNode) (*blockNode, error) { // Dynamically load the previous block from the block database, create // a new block node for it, and update the memory chain accordingly. - prevBlockNode, err := b.loadBlockNode(node.parentHash) - if err != nil { - return nil, err - } - - return prevBlockNode, nil + var prevBlockNode *blockNode + err := b.db.View(func(dbTx database.Tx) error { + var err error + prevBlockNode, err = b.loadBlockNode(dbTx, node.parentHash) + return err + }) + return prevBlockNode, err } // removeBlockNode removes the passed block node from the memory chain by // unlinking all of its children and removing it from the the node and // dependency indices. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) removeBlockNode(node *blockNode) error { if node.parent != nil { - return fmt.Errorf("removeBlockNode must be called with a "+ - " node at the front of the chain - node %v", node.hash) + return AssertError(fmt.Sprintf("removeBlockNode must be "+ + "called with a node at the front of the chain - node %v", + node.hash)) } // Remove the node from the node index. @@ -558,17 +544,14 @@ func (b *BlockChain) removeBlockNode(node *blockNode) error { // and choose the best chain, only a portion of the nodes which form the block // chain are needed in memory. This function walks the chain backwards from the // current best chain to find any nodes before the first needed block node. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) pruneBlockNodes() error { - // Nothing to do if there is not a best chain selected yet. - if b.bestChain == nil { - return nil - } - // Walk the chain backwards to find what should be the new root node. // Intentionally use node.parent instead of getPrevNodeFromNode since // the latter loads the node and the goal is to find nodes still in // memory that can be pruned. - newRootNode := b.bestChain + newRootNode := b.bestNode for i := int32(0); i < minMemoryNodes-1 && newRootNode != nil; i++ { newRootNode = newRootNode.parent } @@ -607,9 +590,9 @@ func (b *BlockChain) pruneBlockNodes() error { // isMajorityVersion determines if a previous number of blocks in the chain // starting with startNode are at least the minimum passed version. -func (b *BlockChain) isMajorityVersion(minVer int32, startNode *blockNode, - numRequired uint64) bool { - +// +// This function MUST be called with the chain state lock held (for writes). +func (b *BlockChain) isMajorityVersion(minVer int32, startNode *blockNode, numRequired uint64) bool { numFound := uint64(0) iterNode := startNode for i := uint64(0); i < b.chainParams.BlockUpgradeNumToCheck && @@ -637,6 +620,8 @@ func (b *BlockChain) isMajorityVersion(minVer int32, startNode *blockNode, // calcPastMedianTime calculates the median time of the previous few blocks // prior to, and including, the passed block node. It is primarily used to // validate new blocks have sane timestamps. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) calcPastMedianTime(startNode *blockNode) (time.Time, error) { // Genesis block. if startNode == nil { @@ -691,9 +676,12 @@ func (b *BlockChain) calcPastMedianTime(startNode *blockNode) (time.Time, error) // prior to, and including, the end of the current best chain. It is primarily // used to ensure new blocks have sane timestamps. // -// This function is NOT safe for concurrent access. +// This function is safe for concurrent access. func (b *BlockChain) CalcPastMedianTime() (time.Time, error) { - return b.calcPastMedianTime(b.bestChain) + b.chainLock.Lock() + defer b.chainLock.Unlock() + + return b.calcPastMedianTime(b.bestNode) } // getReorganizeNodes finds the fork point between the main chain and the passed @@ -703,6 +691,8 @@ func (b *BlockChain) CalcPastMedianTime() (time.Time, error) { // returned list of block nodes) in order to reorganize the chain such that the // passed node is the new end of the main chain. The lists will be empty if the // passed node is not on a side chain. +// +// This function MUST be called with the chain state lock held (for reads). func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List) { // Nothing to detach or attach if there is no node. attachNodes := list.New() @@ -732,7 +722,7 @@ func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List // Start from the end of the main chain and work backwards until the // common ancestor adding each block to the list of nodes to detach from // the main chain. - for n := b.bestChain; n != nil && n.parent != nil; n = n.parent { + for n := b.bestNode; n != nil && n.parent != nil; n = n.parent { if n.hash.IsEqual(ancestor.hash) { break } @@ -744,20 +734,87 @@ func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List // connectBlock handles connecting the passed node/block to the end of the main // (best) chain. -func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block) error { +// +// This passed utxo view must have all referenced txos the block spends marked +// as spent and all of the new txos the block creates added to it. In addition, +// the passed stxos slice must be populated with all of the information for the +// spent txos. This approach is used because the connection validation that +// must happen prior to calling this function requires the same details, so +// it would be inefficient to repeat it. +// +// This function MUST be called with the chain state lock held (for writes). +func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block, view *UtxoViewpoint, stxos []spentTxOut) error { // Make sure it's extending the end of the best chain. prevHash := &block.MsgBlock().Header.PrevBlock - if b.bestChain != nil && !prevHash.IsEqual(b.bestChain.hash) { - return fmt.Errorf("connectBlock must be called with a block " + + if !prevHash.IsEqual(b.bestNode.hash) { + return AssertError("connectBlock must be called with a block " + "that extends the main chain") } - // Insert the block into the database which houses the main chain. - _, err := b.db.InsertBlock(block) + // Sanity check the correct number of stxos are provided. + if len(stxos) != countSpentOutputs(block) { + return AssertError("connectBlock called with inconsistent " + + "spent transaction out information") + } + + // Generate a new best state snapshot that will be used to update the + // database and later memory if all database updates are successful. + b.stateLock.RLock() + curTotalTxns := b.stateSnapshot.TotalTxns + b.stateLock.RUnlock() + numTxns := uint64(len(block.MsgBlock().Transactions)) + blockSize := uint64(block.MsgBlock().SerializeSize()) + state := newBestState(node, blockSize, numTxns, curTotalTxns+numTxns) + + // Atomically insert info into the database. + err := b.db.Update(func(dbTx database.Tx) error { + // Update best block state. + err := dbPutBestState(dbTx, state, node.workSum) + if err != nil { + return err + } + + // Add the block hash and height to the block index which tracks + // the main chain. + err = dbPutBlockIndex(dbTx, block.Sha(), node.height) + if err != nil { + return err + } + + // Update the utxo set using the state of the utxo view. This + // entails removing all of the utxos spent and adding the new + // ones created by the block. + err = dbPutUtxoView(dbTx, view) + if err != nil { + return err + } + + // Update the transaction spend journal by adding a record for + // the block that contains all txos spent by it. + err = dbPutSpendJournalEntry(dbTx, block.Sha(), stxos) + if err != nil { + return err + } + + // Insert the block into the database if it's not already there. + hasBlock, err := dbTx.HasBlock(block.Sha()) + if err != nil { + return err + } + if !hasBlock { + return dbTx.StoreBlock(block) + } + + return nil + }) if err != nil { return err } + // Prune fully spent entries and mark all entries in the view unmodified + // now that the modifications have been committed to the database. + view.commit() + // Add the new node to the memory main chain indices for faster // lookups. node.inMainChain = true @@ -765,68 +822,214 @@ func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block) error { b.depNodes[*prevHash] = append(b.depNodes[*prevHash], node) // This node is now the end of the best chain. - b.bestChain = node + b.bestNode = node + + // Update the state for the best block. Notice how this replaces the + // entire struct instead of updating the existing one. This effectively + // allows the old version to act as a snapshot which callers can use + // freely without needing to hold a lock for the duration. See the + // comments on the state variable for more details. + b.stateLock.Lock() + b.stateSnapshot = state + b.stateLock.Unlock() // Notify the caller that the block was connected to the main chain. // The caller would typically want to react with actions such as // updating wallets. + b.chainLock.Unlock() b.sendNotification(NTBlockConnected, block) + b.chainLock.Lock() return nil } // disconnectBlock handles disconnecting the passed node/block from the end of // the main (best) chain. -func (b *BlockChain) disconnectBlock(node *blockNode, block *btcutil.Block) error { +// +// This function MUST be called with the chain state lock held (for writes). +func (b *BlockChain) disconnectBlock(node *blockNode, block *btcutil.Block, view *UtxoViewpoint) error { // Make sure the node being disconnected is the end of the best chain. - if b.bestChain == nil || !node.hash.IsEqual(b.bestChain.hash) { - return fmt.Errorf("disconnectBlock must be called with the " + + if !node.hash.IsEqual(b.bestNode.hash) { + return AssertError("disconnectBlock must be called with the " + "block at the end of the main chain") } - // Remove the block from the database which houses the main chain. + // Get the previous block node. This function is used over simply + // accessing node.parent directly as it will dynamically create previous + // block nodes as needed. This helps allow only the pieces of the chain + // that are needed to remain in memory. prevNode, err := b.getPrevNodeFromNode(node) if err != nil { return err } - err = b.db.DropAfterBlockBySha(prevNode.hash) + + // Load the previous block since some details for it are needed below. + var prevBlock *btcutil.Block + err = b.db.View(func(dbTx database.Tx) error { + var err error + prevBlock, err = dbFetchBlockByHash(dbTx, prevNode.hash) + return err + }) if err != nil { return err } + // Generate a new best state snapshot that will be used to update the + // database and later memory if all database updates are successful. + b.stateLock.RLock() + curTotalTxns := b.stateSnapshot.TotalTxns + b.stateLock.RUnlock() + numTxns := uint64(len(prevBlock.MsgBlock().Transactions)) + blockSize := uint64(prevBlock.MsgBlock().SerializeSize()) + newTotalTxns := curTotalTxns - uint64(len(block.MsgBlock().Transactions)) + state := newBestState(prevNode, blockSize, numTxns, newTotalTxns) + + err = b.db.Update(func(dbTx database.Tx) error { + // Update best block state. + err := dbPutBestState(dbTx, state, node.workSum) + if err != nil { + return err + } + + // Remove the block hash and height from the block index which + // tracks the main chain. + err = dbRemoveBlockIndex(dbTx, block.Sha(), node.height) + if err != nil { + return err + } + + // Update the utxo set using the state of the utxo view. This + // entails restoring all of the utxos spent and removing the new + // ones created by the block. + err = dbPutUtxoView(dbTx, view) + if err != nil { + return err + } + + // Update the transaction spend journal by removing the record + // that contains all txos spent by the block . + err = dbRemoveSpendJournalEntry(dbTx, block.Sha()) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + // Prune fully spent entries and mark all entries in the view unmodified + // now that the modifications have been committed to the database. + view.commit() + // Put block in the side chain cache. node.inMainChain = false b.blockCache[*node.hash] = block // This node's parent is now the end of the best chain. - b.bestChain = node.parent + b.bestNode = node.parent + + // Update the state for the best block. Notice how this replaces the + // entire struct instead of updating the existing one. This effectively + // allows the old version to act as a snapshot which callers can use + // freely without needing to hold a lock for the duration. See the + // comments on the state variable for more details. + b.stateLock.Lock() + b.stateSnapshot = state + b.stateLock.Unlock() // Notify the caller that the block was disconnected from the main // chain. The caller would typically want to react with actions such as // updating wallets. + b.chainLock.Unlock() b.sendNotification(NTBlockDisconnected, block) + b.chainLock.Lock() return nil } +// countSpentOutputs returns the number of utxos the passed block spends. +func countSpentOutputs(block *btcutil.Block) int { + // Exclude the coinbase transaction since it can't spend anything. + var numSpent int + for _, tx := range block.Transactions()[1:] { + numSpent += len(tx.MsgTx().TxIn) + } + return numSpent +} + // reorganizeChain reorganizes the block chain by disconnecting the nodes in the // detachNodes list and connecting the nodes in the attach list. It expects // that the lists are already in the correct order and are in sync with the // end of the current best chain. Specifically, nodes that are being -// disconnected must be in reverse order (think of popping them off -// the end of the chain) and nodes the are being attached must be in forwards -// order (think pushing them onto the end of the chain). +// disconnected must be in reverse order (think of popping them off the end of +// the chain) and nodes the are being attached must be in forwards order +// (think pushing them onto the end of the chain). // // The flags modify the behavior of this function as follows: // - BFDryRun: Only the checks which ensure the reorganize can be completed // successfully are performed. The chain is not reorganized. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags BehaviorFlags) error { // Ensure all of the needed side chain blocks are in the cache. for e := attachNodes.Front(); e != nil; e = e.Next() { n := e.Value.(*blockNode) if _, exists := b.blockCache[*n.hash]; !exists { - return fmt.Errorf("block %v is missing from the side "+ - "chain block cache", n.hash) + return AssertError(fmt.Sprintf("block %v is missing "+ + "from the side chain block cache", n.hash)) + } + } + + // All of the blocks to detach and related spend journal entries needed + // to unspend transaction outputs in the blocks being disconnected must + // be loaded from the database during the reorg check phase below and + // then they are needed again when doing the actual database updates. + // Rather than doing two loads, cache the loaded data into these slices. + detachBlocks := make([]*btcutil.Block, 0, detachNodes.Len()) + detachSpentTxOuts := make([][]spentTxOut, 0, detachNodes.Len()) + + // Disconnect all of the blocks back to the point of the fork. This + // entails loading the blocks and their associated spent txos from the + // database and using that information to unspend all of the spent txos + // and remove the utxos created by the blocks. + view := NewUtxoViewpoint() + view.SetBestHash(b.bestNode.hash) + for e := detachNodes.Front(); e != nil; e = e.Next() { + n := e.Value.(*blockNode) + var block *btcutil.Block + err := b.db.View(func(dbTx database.Tx) error { + var err error + block, err = dbFetchBlockByHash(dbTx, n.hash) + return err + }) + + // Load all of the utxos referenced by the block that aren't + // already in the view. + err = view.fetchInputUtxos(b.db, block) + if err != nil { + return err + } + + // Load all of the spent txos for the block from the spend + // journal. + var stxos []spentTxOut + err = b.db.View(func(dbTx database.Tx) error { + stxos, err = dbFetchSpendJournalEntry(dbTx, block, view) + return err + }) + if err != nil { + return err + } + + // Store the loaded block and spend journal entry for later. + detachBlocks = append(detachBlocks, block) + detachSpentTxOuts = append(detachSpentTxOuts, stxos) + + err = view.disconnectTransactions(block, stxos) + if err != nil { + return err } } @@ -834,18 +1037,23 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags // to the main chain can be connected without violating any rules and // without actually connecting the block. // - // NOTE: bitcoind does these checks directly when it connects a block. - // The downside to that approach is that if any of these checks fail - // after disconnecting some blocks or attaching others, all of the + // NOTE: These checks could be done directly when connecting a block, + // however the downside to that approach is that if any of these checks + // fail after disconnecting some blocks or attaching others, all of the // operations have to be rolled back to get the chain back into the // state it was before the rule violation (or other failure). There are // at least a couple of ways accomplish that rollback, but both involve - // tweaking the chain. This approach catches these issues before ever - // modifying the chain. + // tweaking the chain and/or database. This approach catches these + // issues before ever modifying the chain. for e := attachNodes.Front(); e != nil; e = e.Next() { n := e.Value.(*blockNode) block := b.blockCache[*n.hash] - err := b.checkConnectBlock(n, block) + + // Notice the spent txout details are not requested here and + // thus will not be generated. This is done because the state + // is not being immediately written to the database, so it is + // not needed. + err := b.checkConnectBlock(n, block, view, nil) if err != nil { return err } @@ -857,14 +1065,35 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags return nil } + // Reset the view for the actual connection code below. This is + // required because the view was previously modified when checking if + // the reorg would be successful and the connection code requires the + // view to be valid from the viewpoint of each block being connected or + // disconnected. + view = NewUtxoViewpoint() + view.SetBestHash(b.bestNode.hash) + // Disconnect blocks from the main chain. - for e := detachNodes.Front(); e != nil; e = e.Next() { + for i, e := 0, detachNodes.Front(); e != nil; i, e = i+1, e.Next() { n := e.Value.(*blockNode) - block, err := b.db.FetchBlockBySha(n.hash) + block := detachBlocks[i] + + // Load all of the utxos referenced by the block that aren't + // already in the view. + err := view.fetchInputUtxos(b.db, block) if err != nil { return err } - err = b.disconnectBlock(n, block) + + // Update the view to unspend all of the spent txos and remove + // the utxos created by the block. + err = view.disconnectTransactions(block, detachSpentTxOuts[i]) + if err != nil { + return err + } + + // Update the database and chain state. + err = b.disconnectBlock(n, block, view) if err != nil { return err } @@ -874,7 +1103,26 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags for e := attachNodes.Front(); e != nil; e = e.Next() { n := e.Value.(*blockNode) block := b.blockCache[*n.hash] - err := b.connectBlock(n, block) + + // Load all of the utxos referenced by the block that aren't + // already in the view. + err := view.fetchInputUtxos(b.db, block) + if err != nil { + return err + } + + // Update the view to mark all utxos referenced by the block + // as spent and add all transactions being created by this block + // to it. Also, provide an stxo slice so the spent txout + // details are generated. + stxos := make([]spentTxOut, 0, countSpentOutputs(block)) + err = view.connectTransactions(block, &stxos) + if err != nil { + return err + } + + // Update the database and chain state. + err = b.connectBlock(n, block, view, stxos) if err != nil { return err } @@ -905,25 +1153,28 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags // cumulatively has the most proof of work. // // The flags modify the behavior of this function as follows: -// - BFFastAdd: Avoids the call to checkConnectBlock which does several -// expensive transaction validation operations. +// - BFFastAdd: Avoids several expensive transaction validation operations. +// This is useful when using checkpoints. // - BFDryRun: Prevents the block from being connected and avoids modifying the // state of the memory chain index. Also, any log messages related to // modifying the state are avoided. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, flags BehaviorFlags) error { fastAdd := flags&BFFastAdd == BFFastAdd dryRun := flags&BFDryRun == BFDryRun - // We haven't selected a best chain yet or we are extending the main - // (best) chain with a new block. This is the most common case. - if b.bestChain == nil || node.parent.hash.IsEqual(b.bestChain.hash) { + // We are extending the main (best) chain with a new block. This is the + // most common case. + if node.parentHash.IsEqual(b.bestNode.hash) { // Perform several checks to verify the block can be connected - // to the main chain (including whatever reorganization might - // be necessary to get this node to the main chain) without - // violating any rules and without actually connecting the - // block. + // to the main chain without violating any rules and without + // actually connecting the block. + view := NewUtxoViewpoint() + view.SetBestHash(node.parentHash) + stxos := make([]spentTxOut, 0, countSpentOutputs(block)) if !fastAdd { - err := b.checkConnectBlock(node, block) + err := b.checkConnectBlock(node, block, view, &stxos) if err != nil { return err } @@ -934,8 +1185,23 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla return nil } + // In the fast add case the code to check the block connection + // was skipped, so the utxo view needs to load the referenced + // utxos, spend them, and add the new utxos being created by + // this block. + if fastAdd { + err := view.fetchInputUtxos(b.db, block) + if err != nil { + return err + } + err = view.connectTransactions(block, &stxos) + if err != nil { + return err + } + } + // Connect the block to the main chain. - err := b.connectBlock(node, block) + err := b.connectBlock(node, block, view, stxos) if err != nil { return err } @@ -981,7 +1247,7 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla // We're extending (or creating) a side chain, but the cumulative // work for this new side chain is not enough to make it the new chain. - if node.workSum.Cmp(b.bestChain.workSum) <= 0 { + if node.workSum.Cmp(b.bestNode.workSum) <= 0 { // Skip Logging info when the dry run flag is set. if dryRun { return nil @@ -1037,24 +1303,22 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla // - Latest block height is after the latest checkpoint (if enabled) // - Latest block has a timestamp newer than 24 hours ago // -// This function is NOT safe for concurrent access. +// This function is safe for concurrent access. func (b *BlockChain) IsCurrent(timeSource MedianTimeSource) bool { - // Not current if there isn't a main (best) chain yet. - if b.bestChain == nil { - return false - } + b.chainLock.RLock() + defer b.chainLock.RUnlock() // Not current if the latest main (best) chain height is before the // latest known good checkpoint (when checkpoints are enabled). - checkpoint := b.LatestCheckpoint() - if checkpoint != nil && b.bestChain.height < checkpoint.Height { + checkpoint := b.latestCheckpoint() + if checkpoint != nil && b.bestNode.height < checkpoint.Height { return false } // Not current if the latest best block has a timestamp before 24 hours // ago. minus24Hours := timeSource.AdjustedTime().Add(-24 * time.Hour) - if b.bestChain.timestamp.Before(minus24Hours) { + if b.bestNode.timestamp.Before(minus24Hours) { return false } @@ -1063,14 +1327,63 @@ func (b *BlockChain) IsCurrent(timeSource MedianTimeSource) bool { return true } -// New returns a BlockChain instance for the passed bitcoin network using the -// provided backing database. It accepts a callback on which notifications -// will be sent when various events take place. See the documentation for -// Notification and NotificationType for details on the types and contents of -// notifications. The provided callback can be nil if the caller is not -// interested in receiving notifications. -func New(db database.Db, params *chaincfg.Params, c NotificationCallback, sigCache *txscript.SigCache) *BlockChain { +// BestSnapshot returns information about the current best chain block and +// related state as of the current point in time. The returned instance must be +// treated as immutable since it is shared by all callers. +// +// This function is safe for concurrent access. +func (b *BlockChain) BestSnapshot() *BestState { + b.stateLock.RLock() + snapshot := b.stateSnapshot + b.stateLock.RUnlock() + return snapshot +} + +// Config is a descriptor which specifies the blockchain instance configuration. +type Config struct { + // DB defines the database which houses the blocks and will be used to + // store all metadata created by this package such as the utxo set. + // + // This field is required. + DB database.DB + + // ChainParams identifies which chain parameters the chain is associated + // with. + // + // This field is required. + ChainParams *chaincfg.Params + + // Notifications defines a callback to which notifications will be sent + // when various events take place. See the documentation for + // Notification and NotificationType for details on the types and + // contents of notifications. + // + // This field can be nil if the caller is not interested in receiving + // notifications. + Notifications NotificationCallback + + // SigCache defines a signature cache to use when when validating + // signatures. This is typically most useful when individual + // transactions are already being validated prior to their inclusion in + // a block such as what is usually done via a transaction memory pool. + // + // This field can be nil if the caller is not interested in using a + // signature cache. + SigCache *txscript.SigCache +} + +// New returns a BlockChain instance using the provided configuration details. +func New(config *Config) (*BlockChain, error) { + // Enforce required config fields. + if config.DB == nil { + return nil, AssertError("blockchain.New database is nil") + } + if config.ChainParams == nil { + return nil, AssertError("blockchain.New chain parameters nil") + } + // Generate a checkpoint by height map from the provided checkpoints. + params := config.ChainParams var checkpointsByHeight map[int32]*chaincfg.Checkpoint if len(params.Checkpoints) > 0 { checkpointsByHeight = make(map[int32]*chaincfg.Checkpoint) @@ -1081,18 +1394,30 @@ func New(db database.Db, params *chaincfg.Params, c NotificationCallback, sigCac } b := BlockChain{ - db: db, - sigCache: sigCache, - chainParams: params, checkpointsByHeight: checkpointsByHeight, - notifications: c, + db: config.DB, + chainParams: params, + notifications: config.Notifications, + sigCache: config.SigCache, root: nil, - bestChain: nil, + bestNode: nil, index: make(map[wire.ShaHash]*blockNode), depNodes: make(map[wire.ShaHash][]*blockNode), orphans: make(map[wire.ShaHash]*orphanBlock), prevOrphans: make(map[wire.ShaHash][]*orphanBlock), blockCache: make(map[wire.ShaHash]*btcutil.Block), } - return &b + + // Initialize the chain state from the passed database. When the db + // does not yet contain any chain state, both it and the chain state + // will be initialized to contain only the genesis block. + if err := b.initChainState(); err != nil { + return nil, err + } + + log.Infof("Chain state (height %d, hash %v, totaltx %d, work %v)", + b.bestNode.height, b.bestNode.hash, b.stateSnapshot.TotalTxns, + b.bestNode.workSum) + + return &b, nil } diff --git a/blockchain/chainio.go b/blockchain/chainio.go new file mode 100644 index 00000000..02869366 --- /dev/null +++ b/blockchain/chainio.go @@ -0,0 +1,1401 @@ +// Copyright (c) 2015-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/big" + "sort" + + database "github.com/btcsuite/btcd/database2" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" +) + +const ( + maxUint32 = 1<<32 - 1 +) + +var ( + // hashIndexBucketName is the name of the db bucket used to house to the + // block hash -> block height index. + hashIndexBucketName = []byte("hashidx") + + // heightIndexBucketName is the name of the db bucket used to house to + // the block height -> block hash index. + heightIndexBucketName = []byte("heightidx") + + // chainStateKeyName is the name of the db key used to store the best + // chain state. + chainStateKeyName = []byte("chainstate") + + // spendJournalBucketName is the name of the db bucket used to house + // transactions outputs that are spent in each block. + spendJournalBucketName = []byte("spendjournal") + + // utxoSetBucketName is the name of the db bucket used to house the + // unspent transaction output set. + utxoSetBucketName = []byte("utxoset") + + // byteOrder is the preferred byte order used for serializing numeric + // fields for storage in the database. + byteOrder = binary.LittleEndian +) + +// errNotInMainChain signifies that a block hash or height that is not in the +// main chain was requested. +type errNotInMainChain string + +// Error implements the error interface. +func (e errNotInMainChain) Error() string { + return string(e) +} + +// isNotInMainChainErr returns whether or not the passed error is an +// errNotInMainChain error. +func isNotInMainChainErr(err error) bool { + _, ok := err.(errNotInMainChain) + return ok +} + +// errDeserialize signifies that a problem was encountered when deserializing +// data. +type errDeserialize string + +// Error implements the error interface. +func (e errDeserialize) Error() string { + return string(e) +} + +// isDeserializeErr returns whether or not the passed error is an errDeserialize +// error. +func isDeserializeErr(err error) bool { + _, ok := err.(errDeserialize) + return ok +} + +// ----------------------------------------------------------------------------- +// The transaction spend journal consists of an entry for each block connected +// to the main chain which contains the transaction outputs the block spends +// serialized such that the order is the reverse of the order they were spent. +// +// This is required because reorganizing the chain necessarily entails +// disconnecting blocks to get back to the point of the fork which implies +// unspending all of the transaction outputs that each block previously spent. +// Since the utxo set, by definition, only contains unspent transaction outputs, +// the spent transaction outputs must be resurrected from somewhere. There is +// more than one way this could be done, however this is the most straight +// forward method that does not require having a transaction index and unpruned +// blockchain. +// +// NOTE: This format is NOT self describing. The additional details such as +// the number of entries (transaction inputs) are expected to come from the +// block itself and the utxo set. The rationale in doing this is to save a +// significant amount of space. This is also the reason the spent outputs are +// serialized in the reverse order they are spent because later transactions +// are allowed to spend outputs from earlier ones in the same block. +// +// The serialized format is: +// +// [
],... +// +// Field Type Size +// header code VLQ variable +// version VLQ variable +// compressed txout +// compressed amount VLQ variable +// compressed script []byte variable +// +// The serialized header code format is: +// bit 0 - containing transaction is a coinbase +// bits 1-x - height of the block that contains the spent txout +// +// NOTE: The header code and version are only encoded when the spent txout was +// the final unspent output of the containing transaction. Otherwise, the +// header code will be 0 and the version is not serialized at all. This is +// done because that information is only needed when the utxo set no longer +// has it. +// +// Example 1: +// From block 170 in main blockchain. +// +// 1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c +// <><><------------------------------------------------------------------> +// | | | +// | version compressed txout +// header code +// +// - header code: 0x13 (coinbase, height 9) +// - transaction version: 1 +// - compressed txout 0: +// - 0x32: VLQ-encoded compressed amount for 5000000000 (50 BTC) +// - 0x05: special script type pay-to-pubkey +// - 0x11...5c: x-coordinate of the pubkey +// +// Example 2: +// Adapted from block 100025 in main blockchain. +// +// 0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e868b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec +// <><----------------------------------------------><----><><----------------------------------------------> +// | | | | | +// | compressed txout | version compressed txout +// header code header code +// +// - Last spent output: +// - header code: 0x00 (was not the final unspent output for containing tx) +// - transaction version: Nothing since header code is 0 +// - compressed txout: +// - 0x91f20f: VLQ-encoded compressed amount for 34405000000 (344.05 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0x6e...86: pubkey hash +// - Second to last spent output: +// - header code: 0x8b9970 (not coinbase, height 100024) +// - transaction version: 1 +// - compressed txout: +// - 0x86c647: VLQ-encoded compressed amount for 13761000000 (137.61 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0xb2...ec: pubkey hash +// ----------------------------------------------------------------------------- + +// spentTxOut contains a spent transaction output and potentially additional +// contextual information such as whether or not it was contained in a coinbase +// transaction, the version of the transaction it was contained in, and which +// block height the containing transaction was included in. As described in +// the comments above, the additional contextual information will only be valid +// when this spent txout is spending the last unspent output of the containing +// transaction. +type spentTxOut struct { + compressed bool // The amount and public key script are compressed. + version int32 // The version of creating tx. + amount int64 // The amount of the output. + pkScript []byte // The public key script for the output. + + // These fields are only set when this is spending the final output of + // the creating tx. + height int32 // Height of the the block containing the creating tx. + isCoinBase bool // Whether creating tx is a coinbase. +} + +// spentTxOutHeaderCode returns the calculated header code to be used when +// serializing the provided stxo entry. +func spentTxOutHeaderCode(stxo *spentTxOut) uint64 { + // The header code is 0 when there is no height set for the stxo. + if stxo.height == 0 { + return 0 + } + + // As described in the serialization format comments, the header code + // encodes the height shifted over one bit and the coinbase flag in the + // lowest bit. + headerCode := uint64(stxo.height) << 1 + if stxo.isCoinBase { + headerCode |= 0x01 + } + + return headerCode +} + +// spentTxOutSerializeSize returns the number of bytes it would take to +// serialize the passed stxo according to the format described above. +func spentTxOutSerializeSize(stxo *spentTxOut) int { + headerCode := spentTxOutHeaderCode(stxo) + size := serializeSizeVLQ(headerCode) + if headerCode != 0 { + size += serializeSizeVLQ(uint64(stxo.version)) + } + return size + compressedTxOutSize(uint64(stxo.amount), stxo.pkScript, + stxo.version, stxo.compressed) +} + +// putSpentTxOut serializes the passed stxo according to the format described +// above directly into the passed target byte slice. The target byte slice must +// be at least large enough to handle the number of bytes returned by the +// spentTxOutSerializeSize function or it will panic. +func putSpentTxOut(target []byte, stxo *spentTxOut) int { + headerCode := spentTxOutHeaderCode(stxo) + offset := putVLQ(target, headerCode) + if headerCode != 0 { + offset += putVLQ(target[offset:], uint64(stxo.version)) + } + return offset + putCompressedTxOut(target[offset:], uint64(stxo.amount), + stxo.pkScript, stxo.version, stxo.compressed) +} + +// decodeSpentTxOut decodes the passed serialized stxo entry, possibly followed +// by other data, into the passed stxo struct. It returns the number of bytes +// read. +// +// Since the serialized stxo entry does not contain the height, version, or +// coinbase flag of the containing transaction when it still has utxos, the +// caller is responsible for passing in the containing transaction version in +// that case. The provided version is ignore when it is serialized as a part of +// the stxo. +// +// An error will be returned if the version is not serialized as a part of the +// stxo and is also not provided to the function. +func decodeSpentTxOut(serialized []byte, stxo *spentTxOut, txVersion int32) (int, error) { + // Ensure there are bytes to decode. + if len(serialized) == 0 { + return 0, errDeserialize("no serialized bytes") + } + + // Deserialize the header code. + code, offset := deserializeVLQ(serialized) + if offset >= len(serialized) { + return offset, errDeserialize("unexpected end of data after " + + "header code") + } + + // Decode the header code and deserialize the containing transaction + // version if needed. + // + // Bit 0 indicates containing transaction is a coinbase. + // Bits 1-x encode height of containing transaction. + if code != 0 { + version, bytesRead := deserializeVLQ(serialized[offset:]) + offset += bytesRead + if offset >= len(serialized) { + return offset, errDeserialize("unexpected end of data " + + "after version") + } + + stxo.isCoinBase = code&0x01 != 0 + stxo.height = int32(code >> 1) + stxo.version = int32(version) + } else { + // Ensure a tx version was specified if the stxo did not encode + // it. This should never happen unless there is database + // corruption or this function is being called without the + // proper state. + if txVersion == 0 { + return offset, AssertError("decodeSpentTxOut called " + + "without a containing tx version when the " + + "serialized stxo that does not encode the " + + "version") + } + stxo.version = txVersion + } + + // Decode the compressed txout. + compAmount, compScript, bytesRead, err := decodeCompressedTxOut( + serialized[offset:], stxo.version) + offset += bytesRead + if err != nil { + return offset, errDeserialize(fmt.Sprintf("unable to decode "+ + "txout: %v", err)) + } + stxo.amount = int64(compAmount) + stxo.pkScript = compScript + stxo.compressed = true + return offset, nil +} + +// deserializeSpendJournalEntry decodes the passed serialized byte slice into a +// slice of spent txouts according to the format described in detail above. +// +// Since the serialization format is not self describing, as noted in the +// format comments, this function also requires the transactions that spend the +// txouts and a utxo view that contains any remaining existing utxos in the +// transactions referenced by the inputs to the passed transasctions. +func deserializeSpendJournalEntry(serialized []byte, txns []*wire.MsgTx, view *UtxoViewpoint) ([]spentTxOut, error) { + // Calculate the total number of stxos. + var numStxos int + for _, tx := range txns { + numStxos += len(tx.TxIn) + } + + // When a block has no spent txouts there is nothing to serialize. + if len(serialized) == 0 { + // Ensure the block actually has no stxos. This should never + // happen unless there is database corruption or an empty entry + // erroneously made its way into the database. + if numStxos != 0 { + return nil, AssertError(fmt.Sprintf("mismatched spend "+ + "journal serialization - no serialization for "+ + "expected %d stxos", numStxos)) + } + + return nil, nil + } + + // Loop backwards through all transactions so everything is read in + // reverse order to match the serialization order. + stxoIdx := numStxos - 1 + stxoInFlight := make(map[wire.ShaHash]int) + offset := 0 + stxos := make([]spentTxOut, numStxos) + for txIdx := len(txns) - 1; txIdx > -1; txIdx-- { + tx := txns[txIdx] + + // Loop backwards through all of the transaction inputs and read + // the associated stxo. + for txInIdx := len(tx.TxIn) - 1; txInIdx > -1; txInIdx-- { + txIn := tx.TxIn[txInIdx] + stxo := &stxos[stxoIdx] + stxoIdx-- + + // Get the transaction version for the stxo based on + // whether or not it should be serialized as a part of + // the stxo. Recall that it is only serialized when the + // stxo spends the final utxo of a transaction. Since + // they are deserialized in reverse order, this means + // the first time an entry for a given containing tx is + // encountered that is not already in the utxo view it + // must have been the final spend and thus the extra + // data will be serialized with the stxo. Otherwise, + // the version must be pulled from the utxo entry. + // + // Since the view is not actually modified as the stxos + // are read here and it's possible later entries + // reference earlier ones, an inflight map is maintained + // to detect this case and pull the tx version from the + // entry that contains the version information as just + // described. + var txVersion int32 + originHash := &txIn.PreviousOutPoint.Hash + entry := view.LookupEntry(originHash) + if entry != nil { + txVersion = entry.Version() + } else if idx, ok := stxoInFlight[*originHash]; ok { + txVersion = stxos[idx].version + } else { + stxoInFlight[*originHash] = stxoIdx + 1 + } + + n, err := decodeSpentTxOut(serialized[offset:], stxo, + txVersion) + offset += n + if err != nil { + return nil, errDeserialize(fmt.Sprintf("unable "+ + "to decode stxo for %v: %v", + txIn.PreviousOutPoint, err)) + } + } + } + + return stxos, nil +} + +// serializeSpendJournalEntry serializes all of the passed spent txouts into a +// single byte slice according to the format described in detail above. +func serializeSpendJournalEntry(stxos []spentTxOut) []byte { + if len(stxos) == 0 { + return nil + } + + // Calculate the size needed to serialize the entire journal entry. + var size int + for i := range stxos { + size += spentTxOutSerializeSize(&stxos[i]) + } + serialized := make([]byte, size) + + // Serialize each individual stxo directly into the slice in reverse + // order one after the other. + var offset int + for i := len(stxos) - 1; i > -1; i-- { + offset += putSpentTxOut(serialized[offset:], &stxos[i]) + } + + return serialized +} + +// dbFetchSpendJournalEntry fetches the spend journal entry for the passed +// block and deserializes it into a slice of spent txout entries. The provided +// view MUST have the utxos referenced by all of the transactions available for +// the passed block since that information is required to reconstruct the spent +// txouts. +func dbFetchSpendJournalEntry(dbTx database.Tx, block *btcutil.Block, view *UtxoViewpoint) ([]spentTxOut, error) { + // Exclude the coinbase transaction since it can't spend anything. + spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName) + serialized := spendBucket.Get(block.Sha()[:]) + blockTxns := block.MsgBlock().Transactions[1:] + stxos, err := deserializeSpendJournalEntry(serialized, blockTxns, view) + if err != nil { + // Ensure any deserialization errors are returned as database + // corruption errors. + if isDeserializeErr(err) { + return nil, database.Error{ + ErrorCode: database.ErrCorruption, + Description: fmt.Sprintf("corrupt spend "+ + "information for %v: %v", block.Sha(), + err), + } + } + + return nil, err + } + + return stxos, nil +} + +// dbPutSpendJournalEntry uses an existing database transaction to update the +// spend journal entry for the given block hash using the provided slice of +// spent txouts. The spent txouts slice must contain an entry for every txout +// the transactions in the block spend in the order they are spent. +func dbPutSpendJournalEntry(dbTx database.Tx, blockHash *wire.ShaHash, stxos []spentTxOut) error { + spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName) + serialized := serializeSpendJournalEntry(stxos) + return spendBucket.Put(blockHash[:], serialized) +} + +// dbRemoveSpendJournalEntry uses an existing database transaction to remove the +// spend journal entry for the passed block hash. +func dbRemoveSpendJournalEntry(dbTx database.Tx, blockHash *wire.ShaHash) error { + spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName) + return spendBucket.Delete(blockHash[:]) +} + +// ----------------------------------------------------------------------------- +// The unspent transaction output (utxo) set consists of an entry for each +// transaction which contains a utxo serialized using a format that is highly +// optimized to reduce space using domain specific compression algorithms. This +// format is a slightly modified version of the format used in Bitcoin Core. +// +// The serialized format is: +// +//
[,...] +// +// Field Type Size +// version VLQ variable +// block height VLQ variable +// header code VLQ variable +// unspentness bitmap []byte variable +// compressed txouts +// compressed amount VLQ variable +// compressed script []byte variable +// +// The serialized header code format is: +// bit 0 - containing transaction is a coinbase +// bit 1 - output zero is unspent +// bit 2 - output one is unspent +// bits 3-x - number of bytes in unspentness bitmap. When both bits 1 and 2 +// are unset, it encodes N-1 since there must be at least one unspent +// output. +// +// The rationale for the header code scheme is as follows: +// - Transactions which only pay to a single output and a change output are +// extremely common, thus an extra byte for the unspentness bitmap can be +// avoided for them by encoding those two outputs in the low order bits. +// - Given it is encoded as a VLQ which can encode values up to 127 with a +// single byte, that leaves 4 bits to represent the number of bytes in the +// unspentness bitmap while still only consuming a single byte for the +// header code. In other words, an unspentness bitmap with up to 120 +// transaction outputs can be encoded with a single-byte header code. +// This covers the vast majority of transactions. +// - Encoding N-1 bytes when both bits 1 and 2 are unset allows an additional +// 8 outpoints to be encoded before causing the header code to require an +// additional byte. +// +// Example 1: +// From tx in main blockchain: +// Blk 1, 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098 +// +// 010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52 +// <><><><------------------------------------------------------------------> +// | | \--------\ | +// | height | compressed txout 0 +// version header code +// +// - version: 1 +// - height: 1 +// - header code: 0x03 (coinbase, output zero unspent, 0 bytes of unspentness) +// - unspentness: Nothing since it is zero bytes +// - compressed txout 0: +// - 0x32: VLQ-encoded compressed amount for 5000000000 (50 BTC) +// - 0x04: special script type pay-to-pubkey +// - 0x96...52: x-coordinate of the pubkey +// +// Example 2: +// From tx in main blockchain: +// Blk 113931, 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f +// +// 0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58 +// <><----><><><------------------------------------------><--------------------------------------------> +// | | | \-------------------\ | | +// version | \--------\ unspentness | compressed txout 2 +// height header code compressed txout 0 +// +// - version: 1 +// - height: 113931 +// - header code: 0x0a (output zero unspent, 1 byte in unspentness bitmap) +// - unspentness: [0x01] (bit 0 is set, so output 0+2 = 2 is unspent) +// NOTE: It's +2 since the first two outputs are encoded in the header code +// - compressed txout 0: +// - 0x12: VLQ-encoded compressed amount for 20000000 (0.2 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0xe2...8a: pubkey hash +// - compressed txout 2: +// - 0x8009: VLQ-encoded compressed amount for 15000000 (0.15 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0xb8...58: pubkey hash +// +// Example 3: +// From tx in main blockchain: +// Blk 338156, 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620 +// +// 0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6 +// <><----><><----><--------------------------------------------------> +// | | | \-----------------\ | +// version | \--------\ unspentness | +// height header code compressed txout 22 +// +// - version: 1 +// - height: 338156 +// - header code: 0x10 (2+1 = 3 bytes in unspentness bitmap) +// NOTE: It's +1 since neither bit 1 nor 2 are set, so N-1 is encoded. +// - unspentness: [0x00 0x00 0x10] (bit 20 is set, so output 20+2 = 22 is unspent) +// NOTE: It's +2 since the first two outputs are encoded in the header code +// - compressed txout 22: +// - 0x8ba5b9e763: VLQ-encoded compressed amount for 366875659 (3.66875659 BTC) +// - 0x01: special script type pay-to-script-hash +// - 0x1d...e6: script hash +// ----------------------------------------------------------------------------- + +// utxoEntryHeaderCode returns the calculated header code to be used when +// serializing the provided utxo entry and the number of bytes needed to encode +// the unspentness bitmap. +func utxoEntryHeaderCode(entry *UtxoEntry, highestOutputIndex uint32) (uint64, int, error) { + // The first two outputs are encoded separately, so offset the index + // accordingly to calculate the correct number of bytes needed to encode + // up to the highest unspent output index. + numBitmapBytes := int((highestOutputIndex + 6) / 8) + + // As previously described, one less than the number of bytes is encoded + // when both output 0 and 1 are spent because there must be at least one + // unspent output. Adjust the number of bytes to encode accordingly and + // encode the value by shifting it over 3 bits. + output0Unspent := !entry.IsOutputSpent(0) + output1Unspent := !entry.IsOutputSpent(1) + var numBitmapBytesAdjustment int + if !output0Unspent && !output1Unspent { + if numBitmapBytes == 0 { + return 0, 0, AssertError("attempt to serialize utxo " + + "header for fully spent transaction") + } + numBitmapBytesAdjustment = 1 + } + headerCode := uint64(numBitmapBytes-numBitmapBytesAdjustment) << 3 + + // Set the coinbase, output 0, and output 1 bits in the header code + // accordingly. + if entry.isCoinBase { + headerCode |= 0x01 // bit 0 + } + if output0Unspent { + headerCode |= 0x02 // bit 1 + } + if output1Unspent { + headerCode |= 0x04 // bit 2 + } + + return headerCode, numBitmapBytes, nil +} + +// serializeUtxoEntry returns the entry serialized to a format that is suitable +// for long-term storage. The format is described in detail above. +func serializeUtxoEntry(entry *UtxoEntry) ([]byte, error) { + // Fully spent entries have no serialization. + if entry.IsFullySpent() { + return nil, nil + } + + // Determine the output order by sorting the sparse output index keys. + outputOrder := make([]int, 0, len(entry.sparseOutputs)) + for outputIndex := range entry.sparseOutputs { + outputOrder = append(outputOrder, int(outputIndex)) + } + sort.Ints(outputOrder) + + // Encode the header code and determine the number of bytes the + // unspentness bitmap needs. + highIndex := uint32(outputOrder[len(outputOrder)-1]) + headerCode, numBitmapBytes, err := utxoEntryHeaderCode(entry, highIndex) + if err != nil { + return nil, err + } + + // The number of bitmap bytes must not exceed a max uint32. This could + // only happen if the number of txouts in a transaction were to exceed + // (2^32 - 1)*8 == (2^35 - 8). This should never happen, but assert + // the condition because if it ever becomes true some distant time in + // the future, the code must be changed. + if numBitmapBytes > maxUint32 { + return nil, AssertError("unspentness bitmap too large to " + + "serialize") + } + + // Calculate the size needed to serialize the entry. + size := serializeSizeVLQ(uint64(entry.version)) + + serializeSizeVLQ(uint64(entry.blockHeight)) + + serializeSizeVLQ(headerCode) + numBitmapBytes + for _, outputIndex := range outputOrder { + out := entry.sparseOutputs[uint32(outputIndex)] + if out.spent { + continue + } + size += compressedTxOutSize(uint64(out.amount), out.pkScript, + entry.version, out.compressed) + } + + // Serialize the version, block height of the containing transaction, + // and header code. + serialized := make([]byte, size) + offset := putVLQ(serialized, uint64(entry.version)) + offset += putVLQ(serialized[offset:], uint64(entry.blockHeight)) + offset += putVLQ(serialized[offset:], headerCode) + + // Serialize the unspentness bitmap. + for i := uint32(0); i < uint32(numBitmapBytes); i++ { + unspentBits := byte(0) + for j := uint32(0); j < 8; j++ { + // The first 2 outputs are encoded via the header code, + // so adjust the output index accordingly. + if !entry.IsOutputSpent(2 + i*8 + j) { + unspentBits |= 1 << uint8(j) + } + } + serialized[offset] = unspentBits + offset++ + } + + // Serialize the compressed unspent transaction outputs. Outputs that + // are already compressed are serialized without modifications. + for _, outputIndex := range outputOrder { + out := entry.sparseOutputs[uint32(outputIndex)] + if out.spent { + continue + } + + offset += putCompressedTxOut(serialized[offset:], + uint64(out.amount), out.pkScript, entry.version, + out.compressed) + } + + return serialized, nil +} + +// deserializeUtxoEntry decodes a utxo entry from the passed serialized byte +// slice into a new UtxoEntry using a format that is suitable for long-term +// storage. The format is described in detail above. +func deserializeUtxoEntry(serialized []byte) (*UtxoEntry, error) { + // Deserialize the version. + version, bytesRead := deserializeVLQ(serialized) + offset := bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after version") + } + + // Deserialize the block height. + blockHeight, bytesRead := deserializeVLQ(serialized[offset:]) + offset += bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after height") + } + + // Deserialize the header code. + code, bytesRead := deserializeVLQ(serialized[offset:]) + offset += bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after header") + } + + // Decode the header code. + // + // Bit 0 indicates whether the containing transaction is a coinbase. + // Bit 1 indicates output 0 is unspent. + // Bit 2 indicates output 1 is unspent. + // Bits 3-x encodes the number of non-zero unspentness bitmap bytes that + // follow. When both output 0 and 1 are spent, it encodes N-1. + isCoinBase := code&0x01 != 0 + output0Unspent := code&0x02 != 0 + output1Unspent := code&0x04 != 0 + numBitmapBytes := code >> 3 + if !output0Unspent && !output1Unspent { + numBitmapBytes++ + } + + // Ensure there are enough bytes left to deserialize the unspentness + // bitmap. + if uint64(len(serialized[offset:])) < numBitmapBytes { + return nil, errDeserialize("unexpected end of data for " + + "unspentness bitmap") + } + + // Create a new utxo entry with the details deserialized above to house + // all of the utxos. + entry := newUtxoEntry(int32(version), isCoinBase, int32(blockHeight)) + + // Add sparse output for unspent outputs 0 and 1 as needed based on the + // details provided by the header code. + var outputIndexes []uint32 + if output0Unspent { + outputIndexes = append(outputIndexes, 0) + } + if output1Unspent { + outputIndexes = append(outputIndexes, 1) + } + + // Decode the unspentness bitmap adding a sparse output for each unspent + // output. + for i := uint32(0); i < uint32(numBitmapBytes); i++ { + unspentBits := serialized[offset] + for j := uint32(0); j < 8; j++ { + if unspentBits&0x01 != 0 { + // The first 2 outputs are encoded via the + // header code, so adjust the output number + // accordingly. + outputNum := 2 + i*8 + j + outputIndexes = append(outputIndexes, outputNum) + } + unspentBits >>= 1 + } + offset++ + } + + // Decode and add all of the utxos. + for i, outputIndex := range outputIndexes { + // Decode the next utxo. The script and amount fields of the + // utxo output are left compressed so decompression can be + // avoided on those that are not accessed. This is done since + // it is quite common for a redeeming transaction to only + // reference a single utxo from a referenced transaction. + compAmount, compScript, bytesRead, err := decodeCompressedTxOut( + serialized[offset:], int32(version)) + if err != nil { + return nil, errDeserialize(fmt.Sprintf("unable to "+ + "decode utxo at index %d: %v", i, err)) + } + offset += bytesRead + + entry.sparseOutputs[outputIndex] = &utxoOutput{ + spent: false, + compressed: true, + pkScript: compScript, + amount: int64(compAmount), + } + } + + return entry, nil +} + +// dbFetchUtxoEntry uses an existing database transaction to fetch all unspent +// outputs for the provided Bitcoin transaction hash from the utxo set. +// +// When there is no entry for the provided hash, nil will be returned for the +// both the entry and the error. +func dbFetchUtxoEntry(dbTx database.Tx, hash *wire.ShaHash) (*UtxoEntry, error) { + // Fetch the unspent transaction output information for the passed + // transaction hash. Return now when there is no entry. + utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) + serializedUtxo := utxoBucket.Get(hash[:]) + if serializedUtxo == nil { + return nil, nil + } + + // A non-nil zero-length entry means there is an entry in the database + // for a fully spent transaction which should never be the case. + if len(serializedUtxo) == 0 { + return nil, AssertError(fmt.Sprintf("database contains entry "+ + "for fully spent tx %v", hash)) + } + + // Deserialize the utxo entry and return it. + entry, err := deserializeUtxoEntry(serializedUtxo) + if err != nil { + // Ensure any deserialization errors are returned as database + // corruption errors. + if isDeserializeErr(err) { + return nil, database.Error{ + ErrorCode: database.ErrCorruption, + Description: fmt.Sprintf("corrupt utxo entry "+ + "for %v: %v", hash, err), + } + } + + return nil, err + } + + return entry, nil +} + +// dbPutUtxoView uses an existing database transaction to update the utxo set +// in the database based on the provided utxo view contents and state. In +// particular, only the entries that have been marked as modified are written +// to the database. +func dbPutUtxoView(dbTx database.Tx, view *UtxoViewpoint) error { + utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) + for txHashIter, entry := range view.entries { + // No need to update the database if the entry was not modified. + if entry == nil || !entry.modified { + continue + } + + // Serialize the utxo entry without any entries that have been + // spent. + serialized, err := serializeUtxoEntry(entry) + if err != nil { + return err + } + + // Make a copy of the hash because the iterator changes on each + // loop iteration and thus slicing it directly would cause the + // data to change out from under the put/delete funcs below. + txHash := txHashIter + + // Remove the utxo entry if it is now fully spent. + if serialized == nil { + if err := utxoBucket.Delete(txHash[:]); err != nil { + return err + } + + continue + } + + // At this point the utxo entry is not fully spent, so store its + // serialization in the database. + err = utxoBucket.Put(txHash[:], serialized) + if err != nil { + return err + } + } + + return nil +} + +// ----------------------------------------------------------------------------- +// The block index consists of two buckets with an entry for every block in the +// main chain. One bucket is for the hash to height mapping and the other is +// for the height to hash mapping. +// +// The serialized format for values in the hash to height bucket is: +// +// +// Field Type Size +// height uint32 4 bytes +// +// The serialized format for values in the height to hash bucket is: +// +// +// Field Type Size +// hash wire.ShaHash wire.HashSize +// ----------------------------------------------------------------------------- + +// dbPutBlockIndex uses an existing database transaction to update or add the +// block index entries for the hash to height and height to hash mappings for +// the provided values. +func dbPutBlockIndex(dbTx database.Tx, hash *wire.ShaHash, height int32) error { + // Serialize the height for use in the index entries. + var serializedHeight [4]byte + byteOrder.PutUint32(serializedHeight[:], uint32(height)) + + // Add the block hash to height mapping to the index. + meta := dbTx.Metadata() + hashIndex := meta.Bucket(hashIndexBucketName) + if err := hashIndex.Put(hash[:], serializedHeight[:]); err != nil { + return err + } + + // Add the block height to hash mapping to the index. + heightIndex := meta.Bucket(heightIndexBucketName) + return heightIndex.Put(serializedHeight[:], hash[:]) +} + +// dbRemoveBlockIndex uses an existing database transaction remove block index +// entries from the hash to height and height to hash mappings for the provided +// values. +func dbRemoveBlockIndex(dbTx database.Tx, hash *wire.ShaHash, height int32) error { + // Remove the block hash to height mapping. + meta := dbTx.Metadata() + hashIndex := meta.Bucket(hashIndexBucketName) + if err := hashIndex.Delete(hash[:]); err != nil { + return err + } + + // Remove the block height to hash mapping. + var serializedHeight [4]byte + byteOrder.PutUint32(serializedHeight[:], uint32(height)) + heightIndex := meta.Bucket(heightIndexBucketName) + return heightIndex.Delete(serializedHeight[:]) +} + +// dbFetchHeightByHash uses an existing database transaction to retrieve the +// height for the provided hash from the index. +func dbFetchHeightByHash(dbTx database.Tx, hash *wire.ShaHash) (int32, error) { + meta := dbTx.Metadata() + hashIndex := meta.Bucket(hashIndexBucketName) + serializedHeight := hashIndex.Get(hash[:]) + if serializedHeight == nil { + str := fmt.Sprintf("block %s is not in the main chain", hash) + return 0, errNotInMainChain(str) + } + + return int32(byteOrder.Uint32(serializedHeight)), nil +} + +// dbFetchHashByHeight uses an existing database transaction to retrieve the +// hash for the provided height from the index. +func dbFetchHashByHeight(dbTx database.Tx, height int32) (*wire.ShaHash, error) { + var serializedHeight [4]byte + byteOrder.PutUint32(serializedHeight[:], uint32(height)) + + meta := dbTx.Metadata() + heightIndex := meta.Bucket(heightIndexBucketName) + hashBytes := heightIndex.Get(serializedHeight[:]) + if hashBytes == nil { + str := fmt.Sprintf("no block at height %d exists", height) + return nil, errNotInMainChain(str) + } + + var hash wire.ShaHash + copy(hash[:], hashBytes) + return &hash, nil +} + +// ----------------------------------------------------------------------------- +// The best chain state consists of the best block hash and height, the total +// number of transactions up to and including those in the best block, and the +// accumulated work sum up to and including the best block. +// +// The serialized format is: +// +// +// +// Field Type Size +// block hash wire.ShaHash wire.HashSize +// block height uint32 4 bytes +// total txns uint64 8 bytes +// work sum length uint32 4 bytes +// work sum big.Int work sum length +// ----------------------------------------------------------------------------- + +// bestChainState represents the data to be stored the database for the current +// best chain state. +type bestChainState struct { + hash wire.ShaHash + height uint32 + totalTxns uint64 + workSum *big.Int +} + +// serializeBestChainState returns the serialization of the passed block best +// chain state. This is data to be stored in the chain state bucket. +func serializeBestChainState(state bestChainState) []byte { + // Calculate the full size needed to serialize the chain state. + workSumBytes := state.workSum.Bytes() + workSumBytesLen := uint32(len(workSumBytes)) + serializedLen := wire.HashSize + 4 + 8 + 4 + workSumBytesLen + + // Serialize the chain state. + serializedData := make([]byte, serializedLen) + copy(serializedData[0:wire.HashSize], state.hash[:]) + offset := uint32(wire.HashSize) + byteOrder.PutUint32(serializedData[offset:], state.height) + offset += 4 + byteOrder.PutUint64(serializedData[offset:], state.totalTxns) + offset += 8 + byteOrder.PutUint32(serializedData[offset:], workSumBytesLen) + offset += 4 + copy(serializedData[offset:], workSumBytes) + return serializedData[:] +} + +// deserializeBestChainState deserializes the passed serialized best chain +// state. This is data stored in the chain state bucket and is updated after +// every block is connected or disconnected form the main chain. +// block. +func deserializeBestChainState(serializedData []byte) (bestChainState, error) { + // Ensure the serialized data has enough bytes to properly deserialize + // the hash, height, total transactions, and work sum length. + if len(serializedData) < wire.HashSize+16 { + return bestChainState{}, database.Error{ + ErrorCode: database.ErrCorruption, + Description: "corrupt best chain state", + } + } + + state := bestChainState{} + copy(state.hash[:], serializedData[0:wire.HashSize]) + offset := uint32(wire.HashSize) + state.height = byteOrder.Uint32(serializedData[offset : offset+4]) + offset += 4 + state.totalTxns = byteOrder.Uint64(serializedData[offset : offset+8]) + offset += 8 + workSumBytesLen := byteOrder.Uint32(serializedData[offset : offset+4]) + offset += 4 + + // Ensure the serialized data has enough bytes to deserialize the work + // sum. + if uint32(len(serializedData[offset:])) < workSumBytesLen { + return bestChainState{}, database.Error{ + ErrorCode: database.ErrCorruption, + Description: "corrupt best chain state", + } + } + workSumBytes := serializedData[offset : offset+workSumBytesLen] + state.workSum = new(big.Int).SetBytes(workSumBytes) + + return state, nil +} + +// dbPutBestState uses an existing database transaction to update the best chain +// state with the given parameters. +func dbPutBestState(dbTx database.Tx, snapshot *BestState, workSum *big.Int) error { + // Serialize the current best chain state. + serializedData := serializeBestChainState(bestChainState{ + hash: *snapshot.Hash, + height: uint32(snapshot.Height), + totalTxns: snapshot.TotalTxns, + workSum: workSum, + }) + + // Store the current best chain state into the database. + return dbTx.Metadata().Put(chainStateKeyName, serializedData) +} + +// createChainState initializes both the database and the chain state to the +// genesis block. This includes creating the necessary buckets and inserting +// the genesis block, so it must only be called on an uninitialized database. +func (b *BlockChain) createChainState() error { + // Create a new node from the genesis block and set it as both the root + // node and the best node. + genesisBlock := btcutil.NewBlock(b.chainParams.GenesisBlock) + header := &genesisBlock.MsgBlock().Header + node := newBlockNode(header, genesisBlock.Sha(), 0) + node.inMainChain = true + b.bestNode = node + b.root = node + + // Add the new node to the index which is used for faster lookups. + b.index[*node.hash] = node + + // Initialize the state related to the best block. + numTxns := uint64(len(genesisBlock.MsgBlock().Transactions)) + blockSize := uint64(genesisBlock.MsgBlock().SerializeSize()) + b.stateSnapshot = newBestState(b.bestNode, blockSize, numTxns, numTxns) + + // Create the initial the database chain state including creating the + // necessary index buckets and inserting the genesis block. + err := b.db.Update(func(dbTx database.Tx) error { + // Create the bucket that houses the chain block hash to height + // index. + meta := dbTx.Metadata() + _, err := meta.CreateBucket(hashIndexBucketName) + if err != nil { + return err + } + + // Create the bucket that houses the chain block height to hash + // index. + _, err = meta.CreateBucket(heightIndexBucketName) + if err != nil { + return err + } + + // Create the bucket that houses the spend journal data. + _, err = meta.CreateBucket(spendJournalBucketName) + if err != nil { + return err + } + + // Create the bucket that houses the utxo set. Note that the + // genesis block coinbase transaction is intentionally not + // inserted here since it is not spendable by consensus rules. + _, err = meta.CreateBucket(utxoSetBucketName) + if err != nil { + return err + } + + // Add the genesis block hash to height and height to hash + // mappings to the index. + err = dbPutBlockIndex(dbTx, b.bestNode.hash, b.bestNode.height) + if err != nil { + return err + } + + // Store the current best chain state into the database. + err = dbPutBestState(dbTx, b.stateSnapshot, b.bestNode.workSum) + if err != nil { + return err + } + + // Store the genesis block into the database. + return dbTx.StoreBlock(genesisBlock) + }) + return err +} + +// initChainState attempts to load and initialize the chain state from the +// database. When the db does not yet contain any chain state, both it and the +// chain state are initialized to the genesis block. +func (b *BlockChain) initChainState() error { + // Attempt to load the chain state from the database. + var isStateInitialized bool + err := b.db.View(func(dbTx database.Tx) error { + // Fetch the stored chain state from the database metadata. + // When it doesn't exist, it means the database hasn't been + // initialized for use with chain yet, so break out now to allow + // that to happen under a writable database transaction. + serializedData := dbTx.Metadata().Get(chainStateKeyName) + if serializedData == nil { + return nil + } + log.Tracef("Serialized chain state: %x", serializedData) + state, err := deserializeBestChainState(serializedData) + if err != nil { + return err + } + + // Load the raw block bytes for the best block. + blockBytes, err := dbTx.FetchBlock(&state.hash) + if err != nil { + return err + } + var block wire.MsgBlock + err = block.Deserialize(bytes.NewReader(blockBytes)) + if err != nil { + return err + } + + // Create a new node and set it as both the root node and the + // best node. The preceding nodes will be loaded on demand as + // needed. + header := &block.Header + node := newBlockNode(header, &state.hash, int32(state.height)) + node.inMainChain = true + node.workSum = state.workSum + b.bestNode = node + b.root = node + + // Add the new node to the indices for faster lookups. + prevHash := node.parentHash + b.index[*node.hash] = node + b.depNodes[*prevHash] = append(b.depNodes[*prevHash], node) + + // Initialize the state related to the best block. + blockSize := uint64(len(blockBytes)) + numTxns := uint64(len(block.Transactions)) + b.stateSnapshot = newBestState(b.bestNode, blockSize, numTxns, + state.totalTxns) + + isStateInitialized = true + return nil + }) + if err != nil { + return err + } + + // There is nothing more to do if the chain state was initialized. + if isStateInitialized { + return nil + } + + // At this point the database has not already been initialized, so + // initialize both it and the chain state to the genesis block. + return b.createChainState() +} + +// dbFetchHeaderByHash uses an existing database transaction to retrieve the +// block header for the provided hash. +func dbFetchHeaderByHash(dbTx database.Tx, hash *wire.ShaHash) (*wire.BlockHeader, error) { + headerBytes, err := dbTx.FetchBlockHeader(hash) + if err != nil { + return nil, err + } + + var header wire.BlockHeader + err = header.Deserialize(bytes.NewReader(headerBytes)) + if err != nil { + return nil, err + } + + return &header, nil +} + +// dbFetchHeaderByHeight uses an existing database transaction to retrieve the +// block header for the provided height. +func dbFetchHeaderByHeight(dbTx database.Tx, height int32) (*wire.BlockHeader, error) { + hash, err := dbFetchHashByHeight(dbTx, height) + if err != nil { + return nil, err + } + + return dbFetchHeaderByHash(dbTx, hash) +} + +// dbFetchBlockByHash uses an existing database transaction to retrieve the raw +// block for the provided hash, deserialize it, retrieve the appropriate height +// from the index, and return a btcutil.Block with the height set. +func dbFetchBlockByHash(dbTx database.Tx, hash *wire.ShaHash) (*btcutil.Block, error) { + // First find the height associated with the provided hash in the index. + blockHeight, err := dbFetchHeightByHash(dbTx, hash) + if err != nil { + return nil, err + } + + // Load the raw block bytes from the database. + blockBytes, err := dbTx.FetchBlock(hash) + if err != nil { + return nil, err + } + + // Create the encapsulated block and set the height appropriately. + block, err := btcutil.NewBlockFromBytes(blockBytes) + if err != nil { + return nil, err + } + block.SetHeight(blockHeight) + + return block, nil +} + +// dbFetchBlockByHeight uses an existing database transaction to retrieve the +// raw block for the provided height, deserialize it, and return a btcutil.Block +// with the height set. +func dbFetchBlockByHeight(dbTx database.Tx, height int32) (*btcutil.Block, error) { + // First find the hash associated with the provided height in the index. + hash, err := dbFetchHashByHeight(dbTx, height) + if err != nil { + return nil, err + } + + // Load the raw block bytes from the database. + blockBytes, err := dbTx.FetchBlock(hash) + if err != nil { + return nil, err + } + + // Create the encapsulated block and set the height appropriately. + block, err := btcutil.NewBlockFromBytes(blockBytes) + if err != nil { + return nil, err + } + block.SetHeight(height) + + return block, nil +} + +// dbMainChainHasBlock uses an existing database transaction to return whether +// or not the main chain contains the block identified by the provided hash. +func dbMainChainHasBlock(dbTx database.Tx, hash *wire.ShaHash) bool { + hashIndex := dbTx.Metadata().Bucket(hashIndexBucketName) + return hashIndex.Get(hash[:]) != nil +} + +// BlockHeightByHash returns the height of the block with the given hash in the +// main chain. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockHeightByHash(hash *wire.ShaHash) (int32, error) { + var height int32 + err := b.db.View(func(dbTx database.Tx) error { + var err error + height, err = dbFetchHeightByHash(dbTx, hash) + return err + }) + return height, err +} + +// BlockHashByHeight returns the hash of the block at the given height in the +// main chain. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockHashByHeight(blockHeight int32) (*wire.ShaHash, error) { + var hash *wire.ShaHash + err := b.db.View(func(dbTx database.Tx) error { + var err error + hash, err = dbFetchHashByHeight(dbTx, blockHeight) + return err + }) + return hash, err +} + +// BlockByHeight returns the block at the given height in the main chain. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockByHeight(blockHeight int32) (*btcutil.Block, error) { + var block *btcutil.Block + err := b.db.View(func(dbTx database.Tx) error { + var err error + block, err = dbFetchBlockByHeight(dbTx, blockHeight) + return err + }) + return block, err +} + +// BlockByHash returns the block from the main chain with the given hash with +// the appropriate chain height set. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockByHash(hash *wire.ShaHash) (*btcutil.Block, error) { + var block *btcutil.Block + err := b.db.View(func(dbTx database.Tx) error { + var err error + block, err = dbFetchBlockByHash(dbTx, hash) + return err + }) + return block, err +} + +// HeightRange returns a range of block hashes for the given start and end +// heights. It is inclusive of the start height and exclusive of the end +// height. The end height will be limited to the current main chain height. +// +// This function is safe for concurrent access. +func (b *BlockChain) HeightRange(startHeight, endHeight int32) ([]wire.ShaHash, error) { + // Ensure requested heights are sane. + if startHeight < 0 { + return nil, fmt.Errorf("start height of fetch range must not "+ + "be less than zero - got %d", startHeight) + } + if endHeight < startHeight { + return nil, fmt.Errorf("end height of fetch range must not "+ + "be less than the start height - got start %d, end %d", + startHeight, endHeight) + } + + // There is nothing to do when the start and end heights are the same, + // so return now to avoid the chain lock and a database transaction. + if startHeight == endHeight { + return nil, nil + } + + // Grab a lock on the chain to prevent it from changing due to a reorg + // while building the hashes. + b.chainLock.RLock() + defer b.chainLock.RUnlock() + + // When the requested start height is after the most recent best chain + // height, there is nothing to do. + latestHeight := b.bestNode.height + if startHeight > latestHeight { + return nil, nil + } + + // Limit the ending height to the latest height of the chain. + if endHeight > latestHeight+1 { + endHeight = latestHeight + 1 + } + + // Fetch as many as are available within the specified range. + var hashList []wire.ShaHash + err := b.db.View(func(dbTx database.Tx) error { + hashes := make([]wire.ShaHash, 0, endHeight-startHeight) + for i := startHeight; i < endHeight; i++ { + hash, err := dbFetchHashByHeight(dbTx, i) + if err != nil { + return err + } + hashes = append(hashes, *hash) + } + + // Set the list to be returned to the constructed list. + hashList = hashes + return nil + }) + return hashList, err +} diff --git a/blockchain/chainio_test.go b/blockchain/chainio_test.go new file mode 100644 index 00000000..fd9a5ebf --- /dev/null +++ b/blockchain/chainio_test.go @@ -0,0 +1,1030 @@ +// Copyright (c) 2015-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "bytes" + "errors" + "math/big" + "reflect" + "testing" + + database "github.com/btcsuite/btcd/database2" + "github.com/btcsuite/btcd/wire" +) + +// TestErrNotInMainChain ensures the functions related to errNotInMainChain work +// as expected. +func TestErrNotInMainChain(t *testing.T) { + errStr := "no block at height 1 exists" + err := error(errNotInMainChain(errStr)) + + // Ensure the stringized output for the error is as expected. + if err.Error() != errStr { + t.Fatalf("errNotInMainChain retuned unexpected error string - "+ + "got %q, want %q", err.Error(), errStr) + } + + // Ensure error is detected as the correct type. + if !isNotInMainChainErr(err) { + t.Fatalf("isNotInMainChainErr did not detect as expected type") + } + err = errors.New("something else") + if isNotInMainChainErr(err) { + t.Fatalf("isNotInMainChainErr detected incorrect type") + } +} + +// maybeDecompress decompresses the amount and public key script fields of the +// stxo and marks it decompressed if needed. +func (o *spentTxOut) maybeDecompress(version int32) { + // Nothing to do if it's not compressed. + if !o.compressed { + return + } + + o.amount = int64(decompressTxOutAmount(uint64(o.amount))) + o.pkScript = decompressScript(o.pkScript, version) + o.compressed = false +} + +// TestStxoSerialization ensures serializing and deserializing spent transaction +// output entries works as expected. +func TestStxoSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stxo spentTxOut + txVersion int32 // When the txout is not fully spent. + serialized []byte + }{ + // From block 170 in main blockchain. + { + name: "Spends last output of coinbase", + stxo: spentTxOut{ + amount: 5000000000, + pkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"), + isCoinBase: true, + height: 9, + version: 1, + }, + serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"), + }, + // Adapted from block 100025 in main blockchain. + { + name: "Spends last output of non coinbase", + stxo: spentTxOut{ + amount: 13761000000, + pkScript: hexToBytes("76a914b2fb57eadf61e106a100a7445a8c3f67898841ec88ac"), + isCoinBase: false, + height: 100024, + version: 1, + }, + serialized: hexToBytes("8b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec"), + }, + // Adapted from block 100025 in main blockchain. + { + name: "Does not spend last output", + stxo: spentTxOut{ + amount: 34405000000, + pkScript: hexToBytes("76a9146edbc6c4d31bae9f1ccc38538a114bf42de65e8688ac"), + version: 1, + }, + txVersion: 1, + serialized: hexToBytes("0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e86"), + }, + } + + for _, test := range tests { + // Ensure the function to calculate the serialized size without + // actually serializing it is calculated properly. + gotSize := spentTxOutSerializeSize(&test.stxo) + if gotSize != len(test.serialized) { + t.Errorf("spentTxOutSerializeSize (%s): did not get "+ + "expected size - got %d, want %d", test.name, + gotSize, len(test.serialized)) + continue + } + + // Ensure the stxo serializes to the expected value. + gotSerialized := make([]byte, gotSize) + gotBytesWritten := putSpentTxOut(gotSerialized, &test.stxo) + if !bytes.Equal(gotSerialized, test.serialized) { + t.Errorf("putSpentTxOut (%s): did not get expected "+ + "bytes - got %x, want %x", test.name, + gotSerialized, test.serialized) + continue + } + if gotBytesWritten != len(test.serialized) { + t.Errorf("putSpentTxOut (%s): did not get expected "+ + "number of bytes written - got %d, want %d", + test.name, gotBytesWritten, + len(test.serialized)) + continue + } + + // Ensure the serialized bytes are decoded back to the expected + // stxo. + var gotStxo spentTxOut + gotBytesRead, err := decodeSpentTxOut(test.serialized, &gotStxo, + test.txVersion) + if err != nil { + t.Errorf("decodeSpentTxOut (%s): unexpected error: %v", + test.name, err) + continue + } + gotStxo.maybeDecompress(test.stxo.version) + if !reflect.DeepEqual(gotStxo, test.stxo) { + t.Errorf("decodeSpentTxOut (%s) mismatched entries - "+ + "got %v, want %v", test.name, gotStxo, test.stxo) + continue + } + if gotBytesRead != len(test.serialized) { + t.Errorf("decodeSpentTxOut (%s): did not get expected "+ + "number of bytes read - got %d, want %d", + test.name, gotBytesRead, len(test.serialized)) + continue + } + } +} + +// TestStxoDecodeErrors performs negative tests against decoding spent +// transaction outputs to ensure error paths work as expected. +func TestStxoDecodeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stxo spentTxOut + txVersion int32 // When the txout is not fully spent. + serialized []byte + bytesRead int // Expected number of bytes read. + errType error + }{ + { + name: "nothing serialized", + stxo: spentTxOut{}, + serialized: hexToBytes(""), + errType: errDeserialize(""), + bytesRead: 0, + }, + { + name: "no data after header code w/o version", + stxo: spentTxOut{}, + serialized: hexToBytes("00"), + errType: errDeserialize(""), + bytesRead: 1, + }, + { + name: "no data after header code with version", + stxo: spentTxOut{}, + serialized: hexToBytes("13"), + errType: errDeserialize(""), + bytesRead: 1, + }, + { + name: "no data after version", + stxo: spentTxOut{}, + serialized: hexToBytes("1301"), + errType: errDeserialize(""), + bytesRead: 2, + }, + { + name: "no serialized tx version and passed 0", + stxo: spentTxOut{}, + serialized: hexToBytes("003205"), + errType: AssertError(""), + bytesRead: 1, + }, + { + name: "incomplete compressed txout", + stxo: spentTxOut{}, + txVersion: 1, + serialized: hexToBytes("0032"), + errType: errDeserialize(""), + bytesRead: 2, + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned. + gotBytesRead, err := decodeSpentTxOut(test.serialized, + &test.stxo, test.txVersion) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("decodeSpentTxOut (%s): expected error type "+ + "does not match - got %T, want %T", test.name, + err, test.errType) + continue + } + + // Ensure the expected number of bytes read is returned. + if gotBytesRead != test.bytesRead { + t.Errorf("decodeSpentTxOut (%s): unexpected number of "+ + "bytes read - got %d, want %d", test.name, + gotBytesRead, test.bytesRead) + continue + } + } +} + +// TestSpendJournalSerialization ensures serializing and deserializing spend +// journal entries works as expected. +func TestSpendJournalSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + entry []spentTxOut + blockTxns []*wire.MsgTx + utxoView *UtxoViewpoint + serialized []byte + }{ + // From block 2 in main blockchain. + { + name: "No spends", + entry: nil, + blockTxns: nil, + utxoView: NewUtxoViewpoint(), + serialized: nil, + }, + // From block 170 in main blockchain. + { + name: "One tx with one input spends last output of coinbase", + entry: []spentTxOut{{ + amount: 5000000000, + pkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"), + isCoinBase: true, + height: 9, + version: 1, + }}, + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9"), + Index: 0, + }, + SignatureScript: hexToBytes("47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901"), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 1000000000, + PkScript: hexToBytes("4104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84cac"), + }, { + Value: 4000000000, + PkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"), + }}, + LockTime: 0, + }}, + utxoView: NewUtxoViewpoint(), + serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"), + }, + // Adapted from block 100025 in main blockchain. + { + name: "Two txns when one spends last output, one doesn't", + entry: []spentTxOut{{ + amount: 34405000000, + pkScript: hexToBytes("76a9146edbc6c4d31bae9f1ccc38538a114bf42de65e8688ac"), + version: 1, + }, { + amount: 13761000000, + pkScript: hexToBytes("76a914b2fb57eadf61e106a100a7445a8c3f67898841ec88ac"), + isCoinBase: false, + height: 100024, + version: 1, + }}, + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"), + Index: 1, + }, + SignatureScript: hexToBytes("493046022100c167eead9840da4a033c9a56470d7794a9bb1605b377ebe5688499b39f94be59022100fb6345cab4324f9ea0b9ee9169337534834638d818129778370f7d378ee4a325014104d962cac5390f12ddb7539507065d0def320d68c040f2e73337c3a1aaaab7195cb5c4d02e0959624d534f3c10c3cf3d73ca5065ebd62ae986b04c6d090d32627c"), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 5000000, + PkScript: hexToBytes("76a914f419b8db4ba65f3b6fcc233acb762ca6f51c23d488ac"), + }, { + Value: 34400000000, + PkScript: hexToBytes("76a914cadf4fc336ab3c6a4610b75f31ba0676b7f663d288ac"), + }}, + LockTime: 0, + }, { + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("92fbe1d4be82f765dfabc9559d4620864b05cc897c4db0e29adac92d294e52b7"), + Index: 0, + }, + SignatureScript: hexToBytes("483045022100e256743154c097465cf13e89955e1c9ff2e55c46051b627751dee0144183157e02201d8d4f02cde8496aae66768f94d35ce54465bd4ae8836004992d3216a93a13f00141049d23ce8686fe9b802a7a938e8952174d35dd2c2089d4112001ed8089023ab4f93a3c9fcd5bfeaa9727858bf640dc1b1c05ec3b434bb59837f8640e8810e87742"), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 5000000, + PkScript: hexToBytes("76a914a983ad7c92c38fc0e2025212e9f972204c6e687088ac"), + }, { + Value: 13756000000, + PkScript: hexToBytes("76a914a6ebd69952ab486a7a300bfffdcb395dc7d47c2388ac"), + }}, + LockTime: 0, + }}, + utxoView: &UtxoViewpoint{entries: map[wire.ShaHash]*UtxoEntry{ + *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"): { + version: 1, + isCoinBase: false, + blockHeight: 100024, + sparseOutputs: map[uint32]*utxoOutput{ + 1: { + amount: 34405000000, + pkScript: hexToBytes("76a9142084541c3931677527a7eafe56fd90207c344eb088ac"), + }, + }, + }, + }}, + serialized: hexToBytes("8b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e86"), + }, + // Hand crafted. + { + name: "One tx, two inputs from same tx, neither spend last output", + entry: []spentTxOut{{ + amount: 165125632, + pkScript: hexToBytes("51"), + version: 1, + }, { + amount: 154370000, + pkScript: hexToBytes("51"), + version: 1, + }}, + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"), + Index: 1, + }, + SignatureScript: hexToBytes(""), + Sequence: 0xffffffff, + }, { + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"), + Index: 2, + }, + SignatureScript: hexToBytes(""), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 165125632, + PkScript: hexToBytes("51"), + }, { + Value: 154370000, + PkScript: hexToBytes("51"), + }}, + LockTime: 0, + }}, + utxoView: &UtxoViewpoint{entries: map[wire.ShaHash]*UtxoEntry{ + *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"): { + version: 1, + isCoinBase: false, + blockHeight: 100000, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 165712179, + pkScript: hexToBytes("51"), + }, + }, + }, + }}, + serialized: hexToBytes("0087bc3707510084c3d19a790751"), + }, + } + + for i, test := range tests { + // Ensure the journal entry serializes to the expected value. + gotBytes := serializeSpendJournalEntry(test.entry) + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeSpendJournalEntry #%d (%s): "+ + "mismatched bytes - got %x, want %x", i, + test.name, gotBytes, test.serialized) + continue + } + + // Deserialize to a spend journal entry. + gotEntry, err := deserializeSpendJournalEntry(test.serialized, + test.blockTxns, test.utxoView) + if err != nil { + t.Errorf("deserializeSpendJournalEntry #%d (%s) "+ + "unexpected error: %v", i, test.name, err) + continue + } + for stxoIdx := range gotEntry { + stxo := &gotEntry[stxoIdx] + stxo.maybeDecompress(test.entry[stxoIdx].version) + } + + // Ensure that the deserialized spend journal entry has the + // correct properties. + if !reflect.DeepEqual(gotEntry, test.entry) { + t.Errorf("deserializeSpendJournalEntry #%d (%s) "+ + "mismatched entries - got %v, want %v", + i, test.name, gotEntry, test.entry) + continue + } + } +} + +// TestSpendJournalErrors performs negative tests against deserializing spend +// journal entries to ensure error paths work as expected. +func TestSpendJournalErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + blockTxns []*wire.MsgTx + utxoView *UtxoViewpoint + serialized []byte + errType error + }{ + // Adapted from block 170 in main blockchain. + { + name: "Force assertion due to missing stxos", + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9"), + Index: 0, + }, + SignatureScript: hexToBytes("47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901"), + Sequence: 0xffffffff, + }}, + LockTime: 0, + }}, + utxoView: NewUtxoViewpoint(), + serialized: hexToBytes(""), + errType: AssertError(""), + }, + { + name: "Force deserialization error in stxos", + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9"), + Index: 0, + }, + SignatureScript: hexToBytes("47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901"), + Sequence: 0xffffffff, + }}, + LockTime: 0, + }}, + utxoView: NewUtxoViewpoint(), + serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a"), + errType: errDeserialize(""), + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned and the returned + // slice is nil. + stxos, err := deserializeSpendJournalEntry(test.serialized, + test.blockTxns, test.utxoView) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("deserializeSpendJournalEntry (%s): expected "+ + "error type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if stxos != nil { + t.Errorf("deserializeSpendJournalEntry (%s): returned "+ + "slice of spent transaction outputs is not nil", + test.name) + continue + } + } +} + +// TestUtxoSerialization ensures serializing and deserializing unspent +// trasaction output entries works as expected. +func TestUtxoSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + entry *UtxoEntry + serialized []byte + }{ + // From tx in main blockchain: + // 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098 + { + name: "Only output 0, coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: true, + blockHeight: 1, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 5000000000, + pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"), + }, + }, + }, + serialized: hexToBytes("010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"), + }, + // From tx in main blockchain: + // 8131ffb0a2c945ecaf9b9063e59558784f9c3a74741ce6ae2a18d0571dac15bb + { + name: "Only output 1, not coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 100001, + sparseOutputs: map[uint32]*utxoOutput{ + 1: { + amount: 1000000, + pkScript: hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"), + }, + }, + }, + serialized: hexToBytes("01858c21040700ee8bd501094a7d5ca318da2506de35e1cb025ddc"), + }, + // Adapted from tx in main blockchain: + // df3f3f442d9699857f7f49de4ff0b5d0f3448bec31cdc7b5bf6d25f2abd637d5 + { + name: "Only output 2, coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: true, + blockHeight: 99004, + sparseOutputs: map[uint32]*utxoOutput{ + 2: { + amount: 100937281, + pkScript: hexToBytes("76a914da33f77cee27c2a975ed5124d7e4f7f97513510188ac"), + }, + }, + }, + serialized: hexToBytes("0185843c010182b095bf4100da33f77cee27c2a975ed5124d7e4f7f975135101"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2 not coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 2: { + amount: 15000000, + pkScript: hexToBytes("76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2, not coinbase, 1 marked spent", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 1: { // This won't be serialized. + spent: true, + amount: 1000000, + pkScript: hexToBytes("76a914e43031c3e46f20bf1ccee9553ce815de5a48467588ac"), + }, + 2: { + amount: 15000000, + pkScript: hexToBytes("76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2, not coinbase, output 2 compressed", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 2: { + // Uncompressed Amount: 15000000 + // Uncompressed PkScript: 76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac + compressed: true, + amount: 137, + pkScript: hexToBytes("00b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2, not coinbase, output 2 compressed, packed indexes reversed", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 2: { + // Uncompressed Amount: 15000000 + // Uncompressed PkScript: 76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac + compressed: true, + amount: 137, + pkScript: hexToBytes("00b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // From tx in main blockchain: + // 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098 + { + name: "Only output 0, coinbase, fully spent", + entry: &UtxoEntry{ + version: 1, + isCoinBase: true, + blockHeight: 1, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + spent: true, + amount: 5000000000, + pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"), + }, + }, + }, + serialized: nil, + }, + // Adapted from tx in main blockchain: + // 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620 + { + name: "Only output 22, not coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 338156, + sparseOutputs: map[uint32]*utxoOutput{ + 22: { + spent: false, + amount: 366875659, + pkScript: hexToBytes("a9141dd46a006572d820e448e12d2bbb38640bc718e687"), + }, + }, + }, + serialized: hexToBytes("0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6"), + }, + } + + for i, test := range tests { + // Ensure the utxo entry serializes to the expected value. + gotBytes, err := serializeUtxoEntry(test.entry) + if err != nil { + t.Errorf("serializeUtxoEntry #%d (%s) unexpected "+ + "error: %v", i, test.name, err) + continue + } + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeUtxoEntry #%d (%s): mismatched "+ + "bytes - got %x, want %x", i, test.name, + gotBytes, test.serialized) + continue + } + + // Don't try to deserialize if the test entry was fully spent + // since it will have a nil serialization. + if test.entry.IsFullySpent() { + continue + } + + // Deserialize to a utxo entry. + utxoEntry, err := deserializeUtxoEntry(test.serialized) + if err != nil { + t.Errorf("deserializeUtxoEntry #%d (%s) unexpected "+ + "error: %v", i, test.name, err) + continue + } + + // Ensure that the deserialized utxo entry has the same + // properties for the containing transaction and block height. + if utxoEntry.Version() != test.entry.Version() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "version: got %d, want %d", i, test.name, + utxoEntry.Version(), test.entry.Version()) + continue + } + if utxoEntry.IsCoinBase() != test.entry.IsCoinBase() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "coinbase flag: got %v, want %v", i, test.name, + utxoEntry.IsCoinBase(), test.entry.IsCoinBase()) + continue + } + if utxoEntry.BlockHeight() != test.entry.BlockHeight() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "block height: got %d, want %d", i, test.name, + utxoEntry.BlockHeight(), + test.entry.BlockHeight()) + continue + } + if utxoEntry.IsFullySpent() != test.entry.IsFullySpent() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "fully spent: got %v, want %v", i, test.name, + utxoEntry.IsFullySpent(), + test.entry.IsFullySpent()) + continue + } + + // Ensure all of the outputs in the test entry match the + // spentness of the output in the deserialized entry and the + // deserialized entry does not contain any additional utxos. + var numUnspent int + for outputIndex := range test.entry.sparseOutputs { + gotSpent := utxoEntry.IsOutputSpent(outputIndex) + wantSpent := test.entry.IsOutputSpent(outputIndex) + if !wantSpent { + numUnspent++ + } + if gotSpent != wantSpent { + t.Errorf("deserializeUtxoEntry #%d (%s) output "+ + "#%d: mismatched spent: got %v, want "+ + "%v", i, test.name, outputIndex, + gotSpent, wantSpent) + continue + + } + } + if len(utxoEntry.sparseOutputs) != numUnspent { + t.Errorf("deserializeUtxoEntry #%d (%s): mismatched "+ + "number of unspent outputs: got %d, want %d", i, + test.name, len(utxoEntry.sparseOutputs), + numUnspent) + continue + } + + // Ensure all of the amounts and scripts of the utxos in the + // deserialized entry match the ones in the test entry. + for outputIndex := range utxoEntry.sparseOutputs { + gotAmount := utxoEntry.AmountByIndex(outputIndex) + wantAmount := test.entry.AmountByIndex(outputIndex) + if gotAmount != wantAmount { + t.Errorf("deserializeUtxoEntry #%d (%s) "+ + "output #%d: mismatched amounts: got "+ + "%d, want %d", i, test.name, + outputIndex, gotAmount, wantAmount) + continue + } + + gotPkScript := utxoEntry.PkScriptByIndex(outputIndex) + wantPkScript := test.entry.PkScriptByIndex(outputIndex) + if !bytes.Equal(gotPkScript, wantPkScript) { + t.Errorf("deserializeUtxoEntry #%d (%s) "+ + "output #%d mismatched scripts: got "+ + "%x, want %x", i, test.name, + outputIndex, gotPkScript, wantPkScript) + continue + } + } + } +} + +// TestUtxoEntryHeaderCodeErrors performs negative tests against unspent +// transaction output header codes to ensure error paths work as expected. +func TestUtxoEntryHeaderCodeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + entry *UtxoEntry + code uint64 + bytesRead int // Expected number of bytes read. + errType error + }{ + { + name: "Force assertion due to fully spent tx", + entry: &UtxoEntry{}, + errType: AssertError(""), + bytesRead: 0, + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned and the code is 0. + code, gotBytesRead, err := utxoEntryHeaderCode(test.entry, 0) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("utxoEntryHeaderCode (%s): expected error "+ + "type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if code != 0 { + t.Errorf("utxoEntryHeaderCode (%s): unexpected code "+ + "on error - got %d, want 0", test.name, code) + continue + } + + // Ensure the expected number of bytes read is returned. + if gotBytesRead != test.bytesRead { + t.Errorf("utxoEntryHeaderCode (%s): unexpected number "+ + "of bytes read - got %d, want %d", test.name, + gotBytesRead, test.bytesRead) + continue + } + } +} + +// TestUtxoEntryDeserializeErrors performs negative tests against deserializing +// unspent transaction outputs to ensure error paths work as expected. +func TestUtxoEntryDeserializeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serialized []byte + errType error + }{ + { + name: "no data after version", + serialized: hexToBytes("01"), + errType: errDeserialize(""), + }, + { + name: "no data after block height", + serialized: hexToBytes("0101"), + errType: errDeserialize(""), + }, + { + name: "no data after header code", + serialized: hexToBytes("010102"), + errType: errDeserialize(""), + }, + { + name: "not enough bytes for unspentness bitmap", + serialized: hexToBytes("01017800"), + errType: errDeserialize(""), + }, + { + name: "incomplete compressed txout", + serialized: hexToBytes("01010232"), + errType: errDeserialize(""), + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned and the returned + // entry is nil. + entry, err := deserializeUtxoEntry(test.serialized) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("deserializeUtxoEntry (%s): expected error "+ + "type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if entry != nil { + t.Errorf("deserializeUtxoEntry (%s): returned entry "+ + "is not nil", test.name) + continue + } + } +} + +// TestBestChainStateSerialization ensures serializing and deserializing the +// best chain state works as expected. +func TestBestChainStateSerialization(t *testing.T) { + t.Parallel() + + workSum := new(big.Int) + tests := []struct { + name string + state bestChainState + serialized []byte + }{ + { + name: "genesis", + state: bestChainState{ + hash: *newShaHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"), + height: 0, + totalTxns: 1, + workSum: func() *big.Int { + workSum.Add(workSum, CalcWork(486604799)) + return new(big.Int).Set(workSum) + }(), // 0x0100010001 + }, + serialized: hexToBytes("6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000000000000100000000000000050000000100010001"), + }, + { + name: "block 1", + state: bestChainState{ + hash: *newShaHashFromStr("00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048"), + height: 1, + totalTxns: 2, + workSum: func() *big.Int { + workSum.Add(workSum, CalcWork(486604799)) + return new(big.Int).Set(workSum) + }(), // 0x0200020002 + }, + serialized: hexToBytes("4860eb18bf1b1620e37e9490fc8a427514416fd75159ab86688e9a8300000000010000000200000000000000050000000200020002"), + }, + } + + for i, test := range tests { + // Ensure the state serializes to the expected value. + gotBytes := serializeBestChainState(test.state) + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeBestChainState #%d (%s): mismatched "+ + "bytes - got %x, want %x", i, test.name, + gotBytes, test.serialized) + continue + } + + // Ensure the serialized bytes are decoded back to the expected + // state. + state, err := deserializeBestChainState(test.serialized) + if err != nil { + t.Errorf("deserializeBestChainState #%d (%s) "+ + "unexpected error: %v", i, test.name, err) + continue + } + if !reflect.DeepEqual(state, test.state) { + t.Errorf("deserializeBestChainState #%d (%s) "+ + "mismatched state - got %v, want %v", i, + test.name, state, test.state) + continue + + } + } +} + +// TestBestChainStateDeserializeErrors performs negative tests against +// deserializing the chain state to ensure error paths work as expected. +func TestBestChainStateDeserializeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serialized []byte + errType error + }{ + { + name: "nothing serialized", + serialized: hexToBytes(""), + errType: database.Error{ErrorCode: database.ErrCorruption}, + }, + { + name: "short data in hash", + serialized: hexToBytes("0000"), + errType: database.Error{ErrorCode: database.ErrCorruption}, + }, + { + name: "short data in work sum", + serialized: hexToBytes("6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000001000000000000000500000001000100"), + errType: database.Error{ErrorCode: database.ErrCorruption}, + }, + } + + for _, test := range tests { + // Ensure the expected error type and code is returned. + _, err := deserializeBestChainState(test.serialized) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("deserializeBestChainState (%s): expected "+ + "error type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if derr, ok := err.(database.Error); ok { + tderr := test.errType.(database.Error) + if derr.ErrorCode != tderr.ErrorCode { + t.Errorf("deserializeBestChainState (%s): "+ + "wrong error code got: %v, want: %v", + test.name, derr.ErrorCode, + tderr.ErrorCode) + continue + } + } + } +} diff --git a/blockchain/checkpoints.go b/blockchain/checkpoints.go index c0dc21c2..b3e49943 100644 --- a/blockchain/checkpoints.go +++ b/blockchain/checkpoints.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -8,6 +8,7 @@ import ( "fmt" "github.com/btcsuite/btcd/chaincfg" + database "github.com/btcsuite/btcd/database2" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" @@ -29,14 +30,23 @@ func newShaHashFromStr(hexStr string) *wire.ShaHash { // DisableCheckpoints provides a mechanism to disable validation against // checkpoints which you DO NOT want to do in production. It is provided only // for debug purposes. +// +// This function is safe for concurrent access. func (b *BlockChain) DisableCheckpoints(disable bool) { + b.chainLock.Lock() b.noCheckpoints = disable + b.chainLock.Unlock() } // Checkpoints returns a slice of checkpoints (regardless of whether they are // already known). When checkpoints are disabled or there are no checkpoints // for the active network, it will return nil. +// +// This function is safe for concurrent access. func (b *BlockChain) Checkpoints() []chaincfg.Checkpoint { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return nil } @@ -44,10 +54,12 @@ func (b *BlockChain) Checkpoints() []chaincfg.Checkpoint { return b.chainParams.Checkpoints } -// LatestCheckpoint returns the most recent checkpoint (regardless of whether it +// latestCheckpoint returns the most recent checkpoint (regardless of whether it // is already known). When checkpoints are disabled or there are no checkpoints // for the active network, it will return nil. -func (b *BlockChain) LatestCheckpoint() *chaincfg.Checkpoint { +// +// This function MUST be called with the chain state lock held (for reads). +func (b *BlockChain) latestCheckpoint() *chaincfg.Checkpoint { if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return nil } @@ -56,9 +68,23 @@ func (b *BlockChain) LatestCheckpoint() *chaincfg.Checkpoint { return &checkpoints[len(checkpoints)-1] } +// LatestCheckpoint returns the most recent checkpoint (regardless of whether it +// is already known). When checkpoints are disabled or there are no checkpoints +// for the active network, it will return nil. +// +// This function is safe for concurrent access. +func (b *BlockChain) LatestCheckpoint() *chaincfg.Checkpoint { + b.chainLock.RLock() + checkpoint := b.latestCheckpoint() + b.chainLock.RUnlock() + return checkpoint +} + // verifyCheckpoint returns whether the passed block height and hash combination // match the hard-coded checkpoint data. It also returns true if there is no // checkpoint data for the passed block height. +// +// This function MUST be called with the chain lock held (for reads). func (b *BlockChain) verifyCheckpoint(height int32, hash *wire.ShaHash) bool { if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return true @@ -83,6 +109,8 @@ func (b *BlockChain) verifyCheckpoint(height int32, hash *wire.ShaHash) bool { // available in the downloaded portion of the block chain and returns the // associated block. It returns nil if a checkpoint can't be found (this should // really only happen for blocks before the first checkpoint). +// +// This function MUST be called with the chain lock held (for reads). func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return nil, nil @@ -98,20 +126,21 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // Perform the initial search to find and cache the latest known // checkpoint if the best chain is not known yet or we haven't already // previously searched. - if b.bestChain == nil || (b.checkpointBlock == nil && b.nextCheckpoint == nil) { + if b.checkpointBlock == nil && b.nextCheckpoint == nil { // Loop backwards through the available checkpoints to find one - // that we already have. + // that is already available. checkpointIndex := -1 - for i := numCheckpoints - 1; i >= 0; i-- { - exists, err := b.db.ExistsSha(checkpoints[i].Hash) - if err != nil { - return nil, err - } - - if exists { - checkpointIndex = i - break + err := b.db.View(func(dbTx database.Tx) error { + for i := numCheckpoints - 1; i >= 0; i-- { + if dbMainChainHasBlock(dbTx, checkpoints[i].Hash) { + checkpointIndex = i + break + } } + return nil + }) + if err != nil { + return nil, err } // No known latest checkpoint. This will only happen on blocks @@ -125,19 +154,26 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // Cache the latest known checkpoint block for future lookups. checkpoint := checkpoints[checkpointIndex] - block, err := b.db.FetchBlockBySha(checkpoint.Hash) + err = b.db.View(func(dbTx database.Tx) error { + block, err := dbFetchBlockByHash(dbTx, checkpoint.Hash) + if err != nil { + return err + } + b.checkpointBlock = block + + // Set the next expected checkpoint block accordingly. + b.nextCheckpoint = nil + if checkpointIndex < numCheckpoints-1 { + b.nextCheckpoint = &checkpoints[checkpointIndex+1] + } + + return nil + }) if err != nil { return nil, err } - b.checkpointBlock = block - // Set the next expected checkpoint block accordingly. - b.nextCheckpoint = nil - if checkpointIndex < numCheckpoints-1 { - b.nextCheckpoint = &checkpoints[checkpointIndex+1] - } - - return block, nil + return b.checkpointBlock, nil } // At this point we've already searched for the latest known checkpoint, @@ -150,7 +186,7 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // When there is a next checkpoint and the height of the current best // chain does not exceed it, the current checkpoint lockin is still // the latest known checkpoint. - if b.bestChain.height < b.nextCheckpoint.Height { + if b.bestNode.height < b.nextCheckpoint.Height { return b.checkpointBlock, nil } @@ -163,11 +199,17 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // that if this lookup fails something is very wrong since the chain // has already passed the checkpoint which was verified as accurate // before inserting it. - block, err := b.db.FetchBlockBySha(b.nextCheckpoint.Hash) + err := b.db.View(func(tx database.Tx) error { + block, err := dbFetchBlockByHash(tx, b.nextCheckpoint.Hash) + if err != nil { + return err + } + b.checkpointBlock = block + return nil + }) if err != nil { return nil, err } - b.checkpointBlock = block // Set the next expected checkpoint. checkpointIndex := -1 @@ -188,8 +230,6 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // isNonstandardTransaction determines whether a transaction contains any // scripts which are not one of the standard types. func isNonstandardTransaction(tx *btcutil.Tx) bool { - // TODO(davec): Should there be checks for the input signature scripts? - // Check all of the output public key scripts for non-standard scripts. for _, txOut := range tx.MsgTx().TxOut { scriptClass := txscript.GetScriptClass(txOut.PkScript) @@ -215,66 +255,80 @@ func isNonstandardTransaction(tx *btcutil.Tx) bool { // // The intent is that candidates are reviewed by a developer to make the final // decision and then manually added to the list of checkpoints for a network. +// +// This function is safe for concurrent access. func (b *BlockChain) IsCheckpointCandidate(block *btcutil.Block) (bool, error) { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + // Checkpoints must be enabled. if b.noCheckpoints { return false, fmt.Errorf("checkpoints are disabled") } - // A checkpoint must be in the main chain. - exists, err := b.db.ExistsSha(block.Sha()) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - - // A checkpoint must be at least CheckpointConfirmations blocks before - // the end of the main chain. - blockHeight := block.Height() - _, mainChainHeight, err := b.db.NewestSha() - if err != nil { - return false, err - } - if blockHeight > (mainChainHeight - CheckpointConfirmations) { - return false, nil - } - - // Get the previous block. - prevHash := &block.MsgBlock().Header.PrevBlock - prevBlock, err := b.db.FetchBlockBySha(prevHash) - if err != nil { - return false, err - } - - // Get the next block. - nextHash, err := b.db.FetchBlockShaByHeight(blockHeight + 1) - if err != nil { - return false, err - } - nextBlock, err := b.db.FetchBlockBySha(nextHash) - if err != nil { - return false, err - } - - // A checkpoint must have timestamps for the block and the blocks on - // either side of it in order (due to the median time allowance this is - // not always the case). - prevTime := prevBlock.MsgBlock().Header.Timestamp - curTime := block.MsgBlock().Header.Timestamp - nextTime := nextBlock.MsgBlock().Header.Timestamp - if prevTime.After(curTime) || nextTime.Before(curTime) { - return false, nil - } - - // A checkpoint must have transactions that only contain standard - // scripts. - for _, tx := range block.Transactions() { - if isNonstandardTransaction(tx) { - return false, nil + var isCandidate bool + err := b.db.View(func(dbTx database.Tx) error { + // A checkpoint must be in the main chain. + blockHeight, err := dbFetchHeightByHash(dbTx, block.Sha()) + if err != nil { + // Only return an error if it's not due to the block not + // being in the main chain. + if !isNotInMainChainErr(err) { + return err + } + return nil } - } - return true, nil + // Ensure the height of the passed block and the entry for the + // block in the main chain match. This should always be the + // case unless the caller provided an invalid block. + if blockHeight != block.Height() { + return fmt.Errorf("passed block height of %d does not "+ + "match the main chain height of %d", + block.Height(), blockHeight) + } + + // A checkpoint must be at least CheckpointConfirmations blocks + // before the end of the main chain. + mainChainHeight := b.bestNode.height + if blockHeight > (mainChainHeight - CheckpointConfirmations) { + return nil + } + + // Get the previous block header. + prevHash := &block.MsgBlock().Header.PrevBlock + prevHeader, err := dbFetchHeaderByHash(dbTx, prevHash) + if err != nil { + return err + } + + // Get the next block header. + nextHeader, err := dbFetchHeaderByHeight(dbTx, blockHeight+1) + if err != nil { + return err + } + + // A checkpoint must have timestamps for the block and the + // blocks on either side of it in order (due to the median time + // allowance this is not always the case). + prevTime := prevHeader.Timestamp + curTime := block.MsgBlock().Header.Timestamp + nextTime := nextHeader.Timestamp + if prevTime.After(curTime) || nextTime.Before(curTime) { + return nil + } + + // A checkpoint must have transactions that only contain + // standard scripts. + for _, tx := range block.Transactions() { + if isNonstandardTransaction(tx) { + return nil + } + } + + // All of the checks passed, so the block is a candidate. + isCandidate = true + return nil + }) + return isCandidate, err } diff --git a/blockchain/common_test.go b/blockchain/common_test.go index 2d8e8c97..b833f820 100644 --- a/blockchain/common_test.go +++ b/blockchain/common_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,18 +15,21 @@ import ( "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/database" - _ "github.com/btcsuite/btcd/database/ldb" - _ "github.com/btcsuite/btcd/database/memdb" + database "github.com/btcsuite/btcd/database2" + _ "github.com/btcsuite/btcd/database2/ffldb" "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcutil" ) -// testDbType is the database backend type to use for the tests. -const testDbType = "memdb" +const ( + // testDbType is the database backend type to use for the tests. + testDbType = "ffldb" -// testDbRoot is the root directory used to create all test databases. -const testDbRoot = "testdbs" + // testDbRoot is the root directory used to create all test databases. + testDbRoot = "testdbs" + + // blockDataNet is the expected network in the test block data. + blockDataNet = wire.MainNet +) // filesExists returns whether or not the named file or directory exists. func fileExists(name string) bool { @@ -41,9 +44,9 @@ func fileExists(name string) bool { // isSupportedDbType returns whether or not the passed database type is // currently supported. func isSupportedDbType(dbType string) bool { - supportedDBs := database.SupportedDBs() - for _, sDbType := range supportedDBs { - if dbType == sDbType { + supportedDrivers := database.SupportedDrivers() + for _, driver := range supportedDrivers { + if dbType == driver { return true } } @@ -61,10 +64,10 @@ func chainSetup(dbName string) (*blockchain.BlockChain, func(), error) { // Handle memory database specially since it doesn't need the disk // specific handling. - var db database.Db + var db database.DB var teardown func() if testDbType == "memdb" { - ndb, err := database.CreateDB(testDbType) + ndb, err := database.Create(testDbType) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } @@ -88,7 +91,7 @@ func chainSetup(dbName string) (*blockchain.BlockChain, func(), error) { // Create a new database to store the accepted blocks into. dbPath := filepath.Join(testDbRoot, dbName) _ = os.RemoveAll(dbPath) - ndb, err := database.CreateDB(testDbType, dbPath) + ndb, err := database.Create(testDbType, dbPath, blockDataNet) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } @@ -97,39 +100,34 @@ func chainSetup(dbName string) (*blockchain.BlockChain, func(), error) { // Setup a teardown function for cleaning up. This function is // returned to the caller to be invoked when it is done testing. teardown = func() { - dbVersionPath := filepath.Join(testDbRoot, dbName+".ver") - db.Sync() db.Close() os.RemoveAll(dbPath) - os.Remove(dbVersionPath) os.RemoveAll(testDbRoot) } } - // Insert the main network genesis block. This is part of the initial - // database setup. - genesisBlock := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock) - _, err := db.InsertBlock(genesisBlock) + // Create the main chain instance. + chain, err := blockchain.New(&blockchain.Config{ + DB: db, + ChainParams: &chaincfg.MainNetParams, + }) if err != nil { teardown() - err := fmt.Errorf("failed to insert genesis block: %v", err) + err := fmt.Errorf("failed to create chain instance: %v", err) return nil, nil, err } - - chain := blockchain.New(db, &chaincfg.MainNetParams, nil, nil) return chain, teardown, nil } -// loadTxStore returns a transaction store loaded from a file. -func loadTxStore(filename string) (blockchain.TxStore, error) { - // The txstore file format is: - // - // +// loadUtxoView returns a utxo view loaded from a file. +func loadUtxoView(filename string) (*blockchain.UtxoViewpoint, error) { + // The utxostore file format is: + // // - // All num and length fields are little-endian uint32s. The spent bits - // field is padded to a byte boundary. + // The serialized utxo len is a little endian uint32 and the serialized + // utxo uses the format described in chainio.go. - filename = filepath.Join("testdata/", filename) + filename = filepath.Join("testdata", filename) fi, err := os.Open(filename) if err != nil { return nil, err @@ -144,80 +142,40 @@ func loadTxStore(filename string) (blockchain.TxStore, error) { } defer fi.Close() - // Num of transaction store objects. - var numItems uint32 - if err := binary.Read(r, binary.LittleEndian, &numItems); err != nil { - return nil, err - } - - txStore := make(blockchain.TxStore) - var uintBuf uint32 - for height := uint32(0); height < numItems; height++ { - txD := blockchain.TxData{} - - // Serialized transaction length. - err = binary.Read(r, binary.LittleEndian, &uintBuf) + view := blockchain.NewUtxoViewpoint() + for { + // Hash of the utxo entry. + var hash wire.ShaHash + _, err := io.ReadAtLeast(r, hash[:], len(hash[:])) if err != nil { - return nil, err - } - serializedTxLen := uintBuf - if serializedTxLen > wire.MaxBlockPayload { - return nil, fmt.Errorf("Read serialized transaction "+ - "length of %d is larger max allowed %d", - serializedTxLen, wire.MaxBlockPayload) - } - - // Transaction. - var msgTx wire.MsgTx - err = msgTx.Deserialize(r) - if err != nil { - return nil, err - } - txD.Tx = btcutil.NewTx(&msgTx) - - // Transaction hash. - txHash := msgTx.TxSha() - txD.Hash = &txHash - - // Block height the transaction came from. - err = binary.Read(r, binary.LittleEndian, &uintBuf) - if err != nil { - return nil, err - } - txD.BlockHeight = int32(uintBuf) - - // Num spent bits. - err = binary.Read(r, binary.LittleEndian, &uintBuf) - if err != nil { - return nil, err - } - numSpentBits := uintBuf - numSpentBytes := numSpentBits / 8 - if numSpentBits%8 != 0 { - numSpentBytes++ - } - - // Packed spent bytes. - spentBytes := make([]byte, numSpentBytes) - _, err = io.ReadFull(r, spentBytes) - if err != nil { - return nil, err - } - - // Populate spent data based on spent bits. - txD.Spent = make([]bool, numSpentBits) - for byteNum, spentByte := range spentBytes { - for bit := 0; bit < 8; bit++ { - if uint32((byteNum*8)+bit) < numSpentBits { - if spentByte&(1< [0x00] +// 127 -> [0x7f] * Max 1-byte value +// 128 -> [0x80 0x00] +// 129 -> [0x80 0x01] +// 255 -> [0x80 0x7f] +// 256 -> [0x81 0x00] +// 16511 -> [0xff 0x7f] * Max 2-byte value +// 16512 -> [0x80 0x80 0x00] +// 32895 -> [0x80 0xff 0x7f] +// 2113663 -> [0xff 0xff 0x7f] * Max 3-byte value +// 270549119 -> [0xff 0xff 0xff 0x7f] * Max 4-byte value +// 2^64-1 -> [0x80 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0x7f] +// +// References: +// https://en.wikipedia.org/wiki/Variable-length_quantity +// http://www.codecodex.com/wiki/Variable-Length_Integers +// ----------------------------------------------------------------------------- + +// serializeSizeVLQ returns the number of bytes it would take to serialize the +// passed number as a variable-length quantity according to the format described +// above. +func serializeSizeVLQ(n uint64) int { + size := 1 + for ; n > 0x7f; n = (n >> 7) - 1 { + size++ + } + + return size +} + +// putVLQ serializes the provided number to a variable-length quantity according +// to the format described above and returns the number of bytes of the encoded +// value. The result is placed directly into the passed byte slice which must +// be at least large enough to handle the number of bytes returned by the +// serializeSizeVLQ function or it will panic. +func putVLQ(target []byte, n uint64) int { + offset := 0 + for ; ; offset++ { + // The high bit is set when another byte follows. + highBitMask := byte(0x80) + if offset == 0 { + highBitMask = 0x00 + } + + target[offset] = byte(n&0x7f) | highBitMask + if n <= 0x7f { + break + } + n = (n >> 7) - 1 + } + + // Reverse the bytes so it is MSB-encoded. + for i, j := 0, offset; i < j; i, j = i+1, j-1 { + target[i], target[j] = target[j], target[i] + } + + return offset + 1 +} + +// deserializeVLQ deserializes the provided variable-length quantity according +// to the format described above. It also returns the number of bytes +// deserialized. +func deserializeVLQ(serialized []byte) (uint64, int) { + var n uint64 + var size int + for _, val := range serialized { + size++ + n = (n << 7) | uint64(val&0x7f) + if val&0x80 != 0x80 { + break + } + n++ + } + + return n, size +} + +// ----------------------------------------------------------------------------- +// In order to reduce the size of stored scripts, a domain specific compression +// algorithm is used which recognizes standard scripts and stores them using +// less bytes than the original script. The compression algorithm used here was +// obtained from Bitcoin Core, so all credits for the algorithm go to it. +// +// The general serialized format is: +// +//