diff --git a/blockchain/accept.go b/blockchain/accept.go index f5dd2842..938d40a8 100644 --- a/blockchain/accept.go +++ b/blockchain/accept.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2015 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -17,6 +17,8 @@ import "github.com/btcsuite/btcutil" // // The flags are also passed to checkBlockContext and connectBestChain. See // their documentation for how the flags modify their behavior. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) error { dryRun := flags&BFDryRun == BFDryRun @@ -74,7 +76,9 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) // chain. The caller would typically want to react by relaying the // inventory to other peers. if !dryRun { + b.chainLock.Unlock() b.sendNotification(NTBlockAccepted, block) + b.chainLock.Lock() } return nil diff --git a/blockchain/blocklocator.go b/blockchain/blocklocator.go index c05de0cf..741813fd 100644 --- a/blockchain/blocklocator.go +++ b/blockchain/blocklocator.go @@ -1,10 +1,11 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package blockchain import ( + database "github.com/btcsuite/btcd/database2" "github.com/btcsuite/btcd/wire" ) @@ -25,7 +26,7 @@ import ( // [17a 16a 15 14 13 12 11 10 9 8 6 2 genesis] type BlockLocator []*wire.ShaHash -// BlockLocatorFromHash returns a block locator for the passed block hash. +// blockLocatorFromHash returns a block locator for the passed block hash. // See BlockLocator for details on the algotirhm used to create a block locator. // // In addition to the general algorithm referenced above, there are a couple of @@ -35,7 +36,9 @@ type BlockLocator []*wire.ShaHash // therefore the block locator will only consist of the genesis hash // - If the passed hash is not currently known, the block locator will only // consist of the passed hash -func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { +// +// This function MUST be called with the chain state lock held (for reads). +func (b *BlockChain) blockLocatorFromHash(hash *wire.ShaHash) BlockLocator { // The locator contains the requested hash at the very least. locator := make(BlockLocator, 0, wire.MaxBlockLocatorsPerMsg) locator = append(locator, hash) @@ -55,7 +58,12 @@ func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { // Try to look up the height for passed block hash. Assume an // error means it doesn't exist and just return the locator for // the block itself. - height, err := b.db.FetchBlockHeightBySha(hash) + var height int32 + err := b.db.View(func(dbTx database.Tx) error { + var err error + height, err = dbFetchHeightByHash(dbTx, hash) + return err + }) if err != nil { return locator } @@ -77,73 +85,94 @@ func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { } // Generate the block locators according to the algorithm described in - // in the BlockLocator comment and make sure to leave room for the - // final genesis hash. - iterNode := node - increment := int32(1) - for len(locator) < wire.MaxBlockLocatorsPerMsg-1 { - // Once there are 10 locators, exponentially increase the - // distance between each block locator. - if len(locator) > 10 { - increment *= 2 - } - blockHeight -= increment - if blockHeight < 1 { - break + // in the BlockLocator comment and make sure to leave room for the final + // genesis hash. + // + // The error is intentionally ignored here since the only way the code + // could fail is if there is something wrong with the database which + // will be caught in short order anyways and it's also safe to ignore + // block locators. + _ = b.db.View(func(dbTx database.Tx) error { + iterNode := node + increment := int32(1) + for len(locator) < wire.MaxBlockLocatorsPerMsg-1 { + // Once there are 10 locators, exponentially increase + // the distance between each block locator. + if len(locator) > 10 { + increment *= 2 + } + blockHeight -= increment + if blockHeight < 1 { + break + } + + // As long as this is still on the side chain, walk + // backwards along the side chain nodes to each block + // height. + if forkHeight != -1 && blockHeight > forkHeight { + // Intentionally use parent field instead of the + // getPrevNodeFromNode function since we don't + // want to dynamically load nodes when building + // block locators. Side chain blocks should + // always be in memory already, and if they + // aren't for some reason it's ok to skip them. + for iterNode != nil && blockHeight > iterNode.height { + iterNode = iterNode.parent + } + if iterNode != nil && iterNode.height == blockHeight { + locator = append(locator, iterNode.hash) + } + continue + } + + // The desired block height is in the main chain, so + // look it up from the main chain database. + h, err := dbFetchHashByHeight(dbTx, blockHeight) + if err != nil { + // This shouldn't happen and it's ok to ignore + // block locators, so just continue to the next + // one. + log.Warnf("Lookup of known valid height failed %v", + blockHeight) + continue + } + locator = append(locator, h) } - // As long as this is still on the side chain, walk backwards - // along the side chain nodes to each block height. - if forkHeight != -1 && blockHeight > forkHeight { - // Intentionally use parent field instead of the - // getPrevNodeFromNode function since we don't want to - // dynamically load nodes when building block locators. - // Side chain blocks should always be in memory already, - // and if they aren't for some reason it's ok to skip - // them. - for iterNode != nil && blockHeight > iterNode.height { - iterNode = iterNode.parent - } - if iterNode != nil && iterNode.height == blockHeight { - locator = append(locator, iterNode.hash) - } - continue - } - - // The desired block height is in the main chain, so look it up - // from the main chain database. - h, err := b.db.FetchBlockShaByHeight(blockHeight) - if err != nil { - // This shouldn't happen and it's ok to ignore block - // locators, so just continue to the next one. - log.Warnf("Lookup of known valid height failed %v", - blockHeight) - continue - } - locator = append(locator, h) - } + return nil + }) // Append the appropriate genesis block. locator = append(locator, b.chainParams.GenesisHash) return locator } +// BlockLocatorFromHash returns a block locator for the passed block hash. +// See BlockLocator for details on the algorithm used to create a block locator. +// +// In addition to the general algorithm referenced above, there are a couple of +// special cases which are handled: +// +// - If the genesis hash is passed, there are no previous hashes to add and +// therefore the block locator will only consist of the genesis hash +// - If the passed hash is not currently known, the block locator will only +// consist of the passed hash +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockLocatorFromHash(hash *wire.ShaHash) BlockLocator { + b.chainLock.RLock() + locator := b.blockLocatorFromHash(hash) + b.chainLock.RUnlock() + return locator +} + // LatestBlockLocator returns a block locator for the latest known tip of the // main (best) chain. +// +// This function is safe for concurrent access. func (b *BlockChain) LatestBlockLocator() (BlockLocator, error) { - // Lookup the latest main chain hash if the best chain hasn't been set - // yet. - if b.bestChain == nil { - // Get the latest block hash for the main chain from the - // database. - hash, _, err := b.db.NewestSha() - if err != nil { - return nil, err - } - - return b.BlockLocatorFromHash(hash), nil - } - - // The best chain is set, so use its hash. - return b.BlockLocatorFromHash(b.bestChain.hash), nil + b.chainLock.RLock() + locator := b.blockLocatorFromHash(b.bestNode.hash) + b.chainLock.RUnlock() + return locator, nil } diff --git a/blockchain/chain.go b/blockchain/chain.go index b4775a59..1026c2e4 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -6,7 +6,6 @@ package blockchain import ( "container/list" - "errors" "fmt" "math/big" "sort" @@ -14,7 +13,7 @@ import ( "time" "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/database" + database "github.com/btcsuite/btcd/database2" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" @@ -32,11 +31,6 @@ const ( minMemoryNodes = BlocksPerRetarget ) -// ErrIndexAlreadyInitialized describes an error that indicates the block index -// is already initialized. -var ErrIndexAlreadyInitialized = errors.New("the block index can only be " + - "initialized before it has been modified") - // blockNode represents a block within the block chain and is primarily used to // aid in selecting the best chain to be the main chain. The main chain is // stored into the block database. @@ -105,21 +99,12 @@ type orphanBlock struct { expiration time.Time } -// addChildrenWork adds the passed work amount to all children all the way -// down the chain. It is used primarily to allow a new node to be dynamically -// inserted from the database into the memory chain prior to nodes we already -// have and update their work values accordingly. -func addChildrenWork(node *blockNode, work *big.Int) { - for _, childNode := range node.children { - childNode.workSum.Add(childNode.workSum, work) - addChildrenWork(childNode, work) - } -} - // removeChildNode deletes node from the provided slice of child block // nodes. It ensures the final pointer reference is set to nil to prevent // potential memory leaks. The original slice is returned unmodified if node // is invalid or not in the slice. +// +// This function MUST be called with the chain state lock held (for writes). func removeChildNode(children []*blockNode, node *blockNode) []*blockNode { if node == nil { return children @@ -138,46 +123,116 @@ func removeChildNode(children []*blockNode, node *blockNode) []*blockNode { return children } +// BestState houses information about the current best block and other info +// related to the state of the main chain as it exists from the point of view of +// the current best block. +// +// The BestSnapshot method can be used to obtain access to this information +// in a concurrent safe manner and the data will not be changed out from under +// the caller when chain state changes occur as the function name implies. +// However, the returned snapshot must be treated as immutable since it is +// shared by all callers. +type BestState struct { + Hash *wire.ShaHash // The hash of the block. + Height int32 // The height of the block. + Bits uint32 // The difficulty bits of the block. + BlockSize uint64 // The size of the block. + NumTxns uint64 // The number of txns in the block. + TotalTxns uint64 // The total number of txns in the chain. +} + +// newBestState returns a new best stats instance for the given parameters. +func newBestState(node *blockNode, blockSize, numTxns, totalTxns uint64) *BestState { + return &BestState{ + Hash: node.hash, + Height: node.height, + Bits: node.bits, + BlockSize: blockSize, + NumTxns: numTxns, + TotalTxns: totalTxns, + } +} + // BlockChain provides functions for working with the bitcoin block chain. // It includes functionality such as rejecting duplicate blocks, ensuring blocks // follow all rules, orphan handling, checkpoint handling, and best chain // selection with reorganization. type BlockChain struct { - db database.Db - chainParams *chaincfg.Params + // The following fields are set when the instance is created and can't + // be changed afterwards, so there is no need to protect them with a + // separate mutex. checkpointsByHeight map[int32]*chaincfg.Checkpoint + db database.DB + chainParams *chaincfg.Params notifications NotificationCallback - root *blockNode - bestChain *blockNode - index map[wire.ShaHash]*blockNode - depNodes map[wire.ShaHash][]*blockNode - orphans map[wire.ShaHash]*orphanBlock - prevOrphans map[wire.ShaHash][]*orphanBlock - oldestOrphan *orphanBlock - orphanLock sync.RWMutex - blockCache map[wire.ShaHash]*btcutil.Block - noVerify bool - noCheckpoints bool - nextCheckpoint *chaincfg.Checkpoint - checkpointBlock *btcutil.Block sigCache *txscript.SigCache + + // chainLock protects concurrent access to the vast majority of the + // fields in this struct below this point. + chainLock sync.RWMutex + + // These fields are configuration parameters that can be toggled at + // runtime. They are protected by the chain lock. + noVerify bool + noCheckpoints bool + + // These fields are related to the memory block index. They are + // protected by the chain lock. + root *blockNode + bestNode *blockNode + index map[wire.ShaHash]*blockNode + depNodes map[wire.ShaHash][]*blockNode + + // These fields are related to handling of orphan blocks. They are + // protected by a combination of the chain lock and the orphan lock. + orphanLock sync.RWMutex + orphans map[wire.ShaHash]*orphanBlock + prevOrphans map[wire.ShaHash][]*orphanBlock + oldestOrphan *orphanBlock + blockCache map[wire.ShaHash]*btcutil.Block + + // These fields are related to checkpoint handling. They are protected + // by the chain lock. + nextCheckpoint *chaincfg.Checkpoint + checkpointBlock *btcutil.Block + + // The state is used as a fairly efficient way to cache information + // about the current best chain state that is returned to callers when + // requested. It operates on the principle of MVCC such that any time a + // new block becomes the best block, the state pointer is replaced with + // a new struct and the old state is left untouched. In this way, + // multiple callers can be pointing to different best chain states. + // This is acceptable for most callers because the state is only being + // queried at a specific point in time. + // + // In addition, some of the fields are stored in the database so the + // chain state can be quickly reconstructed on load. + stateLock sync.RWMutex + stateSnapshot *BestState } // DisableVerify provides a mechanism to disable transaction script validation // which you DO NOT want to do in production as it could allow double spends -// and othe undesirable things. It is provided only for debug purposes since +// and other undesirable things. It is provided only for debug purposes since // script validation is extremely intensive and when debugging it is sometimes // nice to quickly get the chain. +// +// This function is safe for concurrent access. func (b *BlockChain) DisableVerify(disable bool) { + b.chainLock.Lock() b.noVerify = disable + b.chainLock.Unlock() } // HaveBlock returns whether or not the chain instance has the block represented // by the passed hash. This includes checking the various places a block can // be like part of the main chain, on a side chain, or in the orphan pool. // -// This function is NOT safe for concurrent access. +// This function is safe for concurrent access. func (b *BlockChain) HaveBlock(hash *wire.ShaHash) (bool, error) { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + exists, err := b.blockExists(hash) if err != nil { return false, err @@ -319,90 +374,20 @@ func (b *BlockChain) addOrphanBlock(block *btcutil.Block) { return } -// GenerateInitialIndex is an optional function which generates the required -// number of initial block nodes in an optimized fashion. This is optional -// because the memory block index is sparse and previous nodes are dynamically -// loaded as needed. However, during initial startup (when there are no nodes -// in memory yet), dynamically loading all of the required nodes on the fly in -// the usual way is much slower than preloading them. -// -// This function can only be called once and it must be called before any nodes -// are added to the block index. ErrIndexAlreadyInitialized is returned if -// the former is not the case. In practice, this means the function should be -// called directly after New. -func (b *BlockChain) GenerateInitialIndex() error { - // Return an error if the has already been modified. - if b.root != nil { - return ErrIndexAlreadyInitialized - } - - // Grab the latest block height for the main chain from the database. - _, endHeight, err := b.db.NewestSha() - if err != nil { - return err - } - - // Calculate the starting height based on the minimum number of nodes - // needed in memory. - startHeight := endHeight - minMemoryNodes - if startHeight < 0 { - startHeight = 0 - } - - // Loop forwards through each block loading the node into the index for - // the block. - // - // Due to a bug in the SQLite btcdb driver, the FetchBlockBySha call is - // limited to a maximum number of hashes per invocation. Since SQLite - // is going to be nuked eventually, the bug isn't being fixed in the - // driver. In the mean time, work around the issue by calling - // FetchBlockBySha multiple times with the appropriate indices as needed. - for start := startHeight; start <= endHeight; { - hashList, err := b.db.FetchHeightRange(start, endHeight+1) - if err != nil { - return err - } - - // The database did not return any further hashes. Break out of - // the loop now. - if len(hashList) == 0 { - break - } - - // Loop forwards through each block loading the node into the - // index for the block. - for _, hash := range hashList { - // Make a copy of the hash to make sure there are no - // references into the list so it can be freed. - hashCopy := hash - node, err := b.loadBlockNode(&hashCopy) - if err != nil { - return err - } - - // This node is now the end of the best chain. - b.bestChain = node - } - - // Start at the next block after the latest one on the next loop - // iteration. - start += int32(len(hashList)) - } - - return nil -} - // loadBlockNode loads the block identified by hash from the block database, // creates a block node from it, and updates the memory block chain accordingly. -// It is used mainly to dynamically load previous blocks from database as they -// are needed to avoid needing to put the entire block chain in memory. -func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { +// It is used mainly to dynamically load previous blocks from the database as +// they are needed to avoid needing to put the entire block chain in memory. +// +// This function MUST be called with the chain state lock held (for writes). +// The database transaction may be read-only. +func (b *BlockChain) loadBlockNode(dbTx database.Tx, hash *wire.ShaHash) (*blockNode, error) { // Load the block header and height from the db. - blockHeader, err := b.db.FetchBlockHeaderBySha(hash) + blockHeader, err := dbFetchHeaderByHash(dbTx, hash) if err != nil { return nil, err } - blockHeight, err := b.db.FetchBlockHeightBySha(hash) + blockHeight, err := dbFetchHeightByHash(dbTx, hash) if err != nil { return nil, err } @@ -412,14 +397,11 @@ func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { node.inMainChain = true // Add the node to the chain. - // There are several possibilities here: + // There are a few possibilities here: // 1) This node is a child of an existing block node // 2) This node is the parent of one or more nodes - // 3) Neither 1 or 2 is true, and this is not the first node being - // added to the tree which implies it's an orphan block and + // 3) Neither 1 or 2 is true which implies it's an orphan block and // therefore is an error to insert into the chain - // 4) Neither 1 or 2 is true, but this is the first node being added - // to the tree, so it's the root. prevHash := &blockHeader.PrevBlock if parentNode, ok := b.index[*prevHash]; ok { // Case 1 -- This node is a child of an existing block node. @@ -433,28 +415,22 @@ func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { } else if childNodes, ok := b.depNodes[*hash]; ok { // Case 2 -- This node is the parent of one or more nodes. - // Connect this block node to all of its children and update - // all of the children (and their children) with the new work - // sums. + // Update the node's work sum by subtracting this node's work + // from the sum of its first child, and connect the node to all + // of its children. + node.workSum.Sub(childNodes[0].workSum, node.workSum) for _, childNode := range childNodes { childNode.parent = node node.children = append(node.children, childNode) - addChildrenWork(childNode, node.workSum) b.root = node } } else { - // Case 3 -- The node does't have a parent and is not the parent - // of another node. This is only acceptable for the first node - // inserted into the chain. Otherwise it means an arbitrary - // orphan block is trying to be loaded which is not allowed. - if b.root != nil { - str := "loadBlockNode: attempt to insert orphan block %v" - return nil, fmt.Errorf(str, hash) - } - - // Case 4 -- This is the root since it's the first and only node. - b.root = node + // Case 3 -- The node doesn't have a parent and is not the + // parent of another node. This means an arbitrary orphan block + // is trying to be loaded which is not allowed. + str := "loadBlockNode: attempt to insert orphan block %v" + return nil, AssertError(fmt.Sprintf(str, hash)) } // Add the new node to the indices for faster lookups. @@ -467,8 +443,10 @@ func (b *BlockChain) loadBlockNode(hash *wire.ShaHash) (*blockNode, error) { // getPrevNodeFromBlock returns a block node for the block previous to the // passed block (the passed block's parent). When it is already in the memory // block chain, it simply returns it. Otherwise, it loads the previous block -// from the block database, creates a new block node from it, and returns it. -// The returned node will be nil if the genesis block is passed. +// header from the block database, creates a new block node from it, and returns +// it. The returned node will be nil if the genesis block is passed. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) getPrevNodeFromBlock(block *btcutil.Block) (*blockNode, error) { // Genesis block. prevHash := &block.MsgBlock().Header.PrevBlock @@ -483,11 +461,13 @@ func (b *BlockChain) getPrevNodeFromBlock(block *btcutil.Block) (*blockNode, err // Dynamically load the previous block from the block database, create // a new block node for it, and update the memory chain accordingly. - prevBlockNode, err := b.loadBlockNode(prevHash) - if err != nil { - return nil, err - } - return prevBlockNode, nil + var prevBlockNode *blockNode + err := b.db.View(func(dbTx database.Tx) error { + var err error + prevBlockNode, err = b.loadBlockNode(dbTx, prevHash) + return err + }) + return prevBlockNode, err } // getPrevNodeFromNode returns a block node for the block previous to the @@ -497,6 +477,8 @@ func (b *BlockChain) getPrevNodeFromBlock(block *btcutil.Block) (*blockNode, err // to dynamically create a new block node and return it. The memory block // chain is updated accordingly. The returned node will be nil if the genesis // block is passed. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) getPrevNodeFromNode(node *blockNode) (*blockNode, error) { // Return the existing previous block node if it's already there. if node.parent != nil { @@ -510,21 +492,25 @@ func (b *BlockChain) getPrevNodeFromNode(node *blockNode) (*blockNode, error) { // Dynamically load the previous block from the block database, create // a new block node for it, and update the memory chain accordingly. - prevBlockNode, err := b.loadBlockNode(node.parentHash) - if err != nil { - return nil, err - } - - return prevBlockNode, nil + var prevBlockNode *blockNode + err := b.db.View(func(dbTx database.Tx) error { + var err error + prevBlockNode, err = b.loadBlockNode(dbTx, node.parentHash) + return err + }) + return prevBlockNode, err } // removeBlockNode removes the passed block node from the memory chain by // unlinking all of its children and removing it from the the node and // dependency indices. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) removeBlockNode(node *blockNode) error { if node.parent != nil { - return fmt.Errorf("removeBlockNode must be called with a "+ - " node at the front of the chain - node %v", node.hash) + return AssertError(fmt.Sprintf("removeBlockNode must be "+ + "called with a node at the front of the chain - node %v", + node.hash)) } // Remove the node from the node index. @@ -558,17 +544,14 @@ func (b *BlockChain) removeBlockNode(node *blockNode) error { // and choose the best chain, only a portion of the nodes which form the block // chain are needed in memory. This function walks the chain backwards from the // current best chain to find any nodes before the first needed block node. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) pruneBlockNodes() error { - // Nothing to do if there is not a best chain selected yet. - if b.bestChain == nil { - return nil - } - // Walk the chain backwards to find what should be the new root node. // Intentionally use node.parent instead of getPrevNodeFromNode since // the latter loads the node and the goal is to find nodes still in // memory that can be pruned. - newRootNode := b.bestChain + newRootNode := b.bestNode for i := int32(0); i < minMemoryNodes-1 && newRootNode != nil; i++ { newRootNode = newRootNode.parent } @@ -607,9 +590,9 @@ func (b *BlockChain) pruneBlockNodes() error { // isMajorityVersion determines if a previous number of blocks in the chain // starting with startNode are at least the minimum passed version. -func (b *BlockChain) isMajorityVersion(minVer int32, startNode *blockNode, - numRequired uint64) bool { - +// +// This function MUST be called with the chain state lock held (for writes). +func (b *BlockChain) isMajorityVersion(minVer int32, startNode *blockNode, numRequired uint64) bool { numFound := uint64(0) iterNode := startNode for i := uint64(0); i < b.chainParams.BlockUpgradeNumToCheck && @@ -637,6 +620,8 @@ func (b *BlockChain) isMajorityVersion(minVer int32, startNode *blockNode, // calcPastMedianTime calculates the median time of the previous few blocks // prior to, and including, the passed block node. It is primarily used to // validate new blocks have sane timestamps. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) calcPastMedianTime(startNode *blockNode) (time.Time, error) { // Genesis block. if startNode == nil { @@ -691,9 +676,12 @@ func (b *BlockChain) calcPastMedianTime(startNode *blockNode) (time.Time, error) // prior to, and including, the end of the current best chain. It is primarily // used to ensure new blocks have sane timestamps. // -// This function is NOT safe for concurrent access. +// This function is safe for concurrent access. func (b *BlockChain) CalcPastMedianTime() (time.Time, error) { - return b.calcPastMedianTime(b.bestChain) + b.chainLock.Lock() + defer b.chainLock.Unlock() + + return b.calcPastMedianTime(b.bestNode) } // getReorganizeNodes finds the fork point between the main chain and the passed @@ -703,6 +691,8 @@ func (b *BlockChain) CalcPastMedianTime() (time.Time, error) { // returned list of block nodes) in order to reorganize the chain such that the // passed node is the new end of the main chain. The lists will be empty if the // passed node is not on a side chain. +// +// This function MUST be called with the chain state lock held (for reads). func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List) { // Nothing to detach or attach if there is no node. attachNodes := list.New() @@ -732,7 +722,7 @@ func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List // Start from the end of the main chain and work backwards until the // common ancestor adding each block to the list of nodes to detach from // the main chain. - for n := b.bestChain; n != nil && n.parent != nil; n = n.parent { + for n := b.bestNode; n != nil && n.parent != nil; n = n.parent { if n.hash.IsEqual(ancestor.hash) { break } @@ -744,20 +734,87 @@ func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List // connectBlock handles connecting the passed node/block to the end of the main // (best) chain. -func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block) error { +// +// This passed utxo view must have all referenced txos the block spends marked +// as spent and all of the new txos the block creates added to it. In addition, +// the passed stxos slice must be populated with all of the information for the +// spent txos. This approach is used because the connection validation that +// must happen prior to calling this function requires the same details, so +// it would be inefficient to repeat it. +// +// This function MUST be called with the chain state lock held (for writes). +func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block, view *UtxoViewpoint, stxos []spentTxOut) error { // Make sure it's extending the end of the best chain. prevHash := &block.MsgBlock().Header.PrevBlock - if b.bestChain != nil && !prevHash.IsEqual(b.bestChain.hash) { - return fmt.Errorf("connectBlock must be called with a block " + + if !prevHash.IsEqual(b.bestNode.hash) { + return AssertError("connectBlock must be called with a block " + "that extends the main chain") } - // Insert the block into the database which houses the main chain. - _, err := b.db.InsertBlock(block) + // Sanity check the correct number of stxos are provided. + if len(stxos) != countSpentOutputs(block) { + return AssertError("connectBlock called with inconsistent " + + "spent transaction out information") + } + + // Generate a new best state snapshot that will be used to update the + // database and later memory if all database updates are successful. + b.stateLock.RLock() + curTotalTxns := b.stateSnapshot.TotalTxns + b.stateLock.RUnlock() + numTxns := uint64(len(block.MsgBlock().Transactions)) + blockSize := uint64(block.MsgBlock().SerializeSize()) + state := newBestState(node, blockSize, numTxns, curTotalTxns+numTxns) + + // Atomically insert info into the database. + err := b.db.Update(func(dbTx database.Tx) error { + // Update best block state. + err := dbPutBestState(dbTx, state, node.workSum) + if err != nil { + return err + } + + // Add the block hash and height to the block index which tracks + // the main chain. + err = dbPutBlockIndex(dbTx, block.Sha(), node.height) + if err != nil { + return err + } + + // Update the utxo set using the state of the utxo view. This + // entails removing all of the utxos spent and adding the new + // ones created by the block. + err = dbPutUtxoView(dbTx, view) + if err != nil { + return err + } + + // Update the transaction spend journal by adding a record for + // the block that contains all txos spent by it. + err = dbPutSpendJournalEntry(dbTx, block.Sha(), stxos) + if err != nil { + return err + } + + // Insert the block into the database if it's not already there. + hasBlock, err := dbTx.HasBlock(block.Sha()) + if err != nil { + return err + } + if !hasBlock { + return dbTx.StoreBlock(block) + } + + return nil + }) if err != nil { return err } + // Prune fully spent entries and mark all entries in the view unmodified + // now that the modifications have been committed to the database. + view.commit() + // Add the new node to the memory main chain indices for faster // lookups. node.inMainChain = true @@ -765,68 +822,214 @@ func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block) error { b.depNodes[*prevHash] = append(b.depNodes[*prevHash], node) // This node is now the end of the best chain. - b.bestChain = node + b.bestNode = node + + // Update the state for the best block. Notice how this replaces the + // entire struct instead of updating the existing one. This effectively + // allows the old version to act as a snapshot which callers can use + // freely without needing to hold a lock for the duration. See the + // comments on the state variable for more details. + b.stateLock.Lock() + b.stateSnapshot = state + b.stateLock.Unlock() // Notify the caller that the block was connected to the main chain. // The caller would typically want to react with actions such as // updating wallets. + b.chainLock.Unlock() b.sendNotification(NTBlockConnected, block) + b.chainLock.Lock() return nil } // disconnectBlock handles disconnecting the passed node/block from the end of // the main (best) chain. -func (b *BlockChain) disconnectBlock(node *blockNode, block *btcutil.Block) error { +// +// This function MUST be called with the chain state lock held (for writes). +func (b *BlockChain) disconnectBlock(node *blockNode, block *btcutil.Block, view *UtxoViewpoint) error { // Make sure the node being disconnected is the end of the best chain. - if b.bestChain == nil || !node.hash.IsEqual(b.bestChain.hash) { - return fmt.Errorf("disconnectBlock must be called with the " + + if !node.hash.IsEqual(b.bestNode.hash) { + return AssertError("disconnectBlock must be called with the " + "block at the end of the main chain") } - // Remove the block from the database which houses the main chain. + // Get the previous block node. This function is used over simply + // accessing node.parent directly as it will dynamically create previous + // block nodes as needed. This helps allow only the pieces of the chain + // that are needed to remain in memory. prevNode, err := b.getPrevNodeFromNode(node) if err != nil { return err } - err = b.db.DropAfterBlockBySha(prevNode.hash) + + // Load the previous block since some details for it are needed below. + var prevBlock *btcutil.Block + err = b.db.View(func(dbTx database.Tx) error { + var err error + prevBlock, err = dbFetchBlockByHash(dbTx, prevNode.hash) + return err + }) if err != nil { return err } + // Generate a new best state snapshot that will be used to update the + // database and later memory if all database updates are successful. + b.stateLock.RLock() + curTotalTxns := b.stateSnapshot.TotalTxns + b.stateLock.RUnlock() + numTxns := uint64(len(prevBlock.MsgBlock().Transactions)) + blockSize := uint64(prevBlock.MsgBlock().SerializeSize()) + newTotalTxns := curTotalTxns - uint64(len(block.MsgBlock().Transactions)) + state := newBestState(prevNode, blockSize, numTxns, newTotalTxns) + + err = b.db.Update(func(dbTx database.Tx) error { + // Update best block state. + err := dbPutBestState(dbTx, state, node.workSum) + if err != nil { + return err + } + + // Remove the block hash and height from the block index which + // tracks the main chain. + err = dbRemoveBlockIndex(dbTx, block.Sha(), node.height) + if err != nil { + return err + } + + // Update the utxo set using the state of the utxo view. This + // entails restoring all of the utxos spent and removing the new + // ones created by the block. + err = dbPutUtxoView(dbTx, view) + if err != nil { + return err + } + + // Update the transaction spend journal by removing the record + // that contains all txos spent by the block . + err = dbRemoveSpendJournalEntry(dbTx, block.Sha()) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + // Prune fully spent entries and mark all entries in the view unmodified + // now that the modifications have been committed to the database. + view.commit() + // Put block in the side chain cache. node.inMainChain = false b.blockCache[*node.hash] = block // This node's parent is now the end of the best chain. - b.bestChain = node.parent + b.bestNode = node.parent + + // Update the state for the best block. Notice how this replaces the + // entire struct instead of updating the existing one. This effectively + // allows the old version to act as a snapshot which callers can use + // freely without needing to hold a lock for the duration. See the + // comments on the state variable for more details. + b.stateLock.Lock() + b.stateSnapshot = state + b.stateLock.Unlock() // Notify the caller that the block was disconnected from the main // chain. The caller would typically want to react with actions such as // updating wallets. + b.chainLock.Unlock() b.sendNotification(NTBlockDisconnected, block) + b.chainLock.Lock() return nil } +// countSpentOutputs returns the number of utxos the passed block spends. +func countSpentOutputs(block *btcutil.Block) int { + // Exclude the coinbase transaction since it can't spend anything. + var numSpent int + for _, tx := range block.Transactions()[1:] { + numSpent += len(tx.MsgTx().TxIn) + } + return numSpent +} + // reorganizeChain reorganizes the block chain by disconnecting the nodes in the // detachNodes list and connecting the nodes in the attach list. It expects // that the lists are already in the correct order and are in sync with the // end of the current best chain. Specifically, nodes that are being -// disconnected must be in reverse order (think of popping them off -// the end of the chain) and nodes the are being attached must be in forwards -// order (think pushing them onto the end of the chain). +// disconnected must be in reverse order (think of popping them off the end of +// the chain) and nodes the are being attached must be in forwards order +// (think pushing them onto the end of the chain). // // The flags modify the behavior of this function as follows: // - BFDryRun: Only the checks which ensure the reorganize can be completed // successfully are performed. The chain is not reorganized. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags BehaviorFlags) error { // Ensure all of the needed side chain blocks are in the cache. for e := attachNodes.Front(); e != nil; e = e.Next() { n := e.Value.(*blockNode) if _, exists := b.blockCache[*n.hash]; !exists { - return fmt.Errorf("block %v is missing from the side "+ - "chain block cache", n.hash) + return AssertError(fmt.Sprintf("block %v is missing "+ + "from the side chain block cache", n.hash)) + } + } + + // All of the blocks to detach and related spend journal entries needed + // to unspend transaction outputs in the blocks being disconnected must + // be loaded from the database during the reorg check phase below and + // then they are needed again when doing the actual database updates. + // Rather than doing two loads, cache the loaded data into these slices. + detachBlocks := make([]*btcutil.Block, 0, detachNodes.Len()) + detachSpentTxOuts := make([][]spentTxOut, 0, detachNodes.Len()) + + // Disconnect all of the blocks back to the point of the fork. This + // entails loading the blocks and their associated spent txos from the + // database and using that information to unspend all of the spent txos + // and remove the utxos created by the blocks. + view := NewUtxoViewpoint() + view.SetBestHash(b.bestNode.hash) + for e := detachNodes.Front(); e != nil; e = e.Next() { + n := e.Value.(*blockNode) + var block *btcutil.Block + err := b.db.View(func(dbTx database.Tx) error { + var err error + block, err = dbFetchBlockByHash(dbTx, n.hash) + return err + }) + + // Load all of the utxos referenced by the block that aren't + // already in the view. + err = view.fetchInputUtxos(b.db, block) + if err != nil { + return err + } + + // Load all of the spent txos for the block from the spend + // journal. + var stxos []spentTxOut + err = b.db.View(func(dbTx database.Tx) error { + stxos, err = dbFetchSpendJournalEntry(dbTx, block, view) + return err + }) + if err != nil { + return err + } + + // Store the loaded block and spend journal entry for later. + detachBlocks = append(detachBlocks, block) + detachSpentTxOuts = append(detachSpentTxOuts, stxos) + + err = view.disconnectTransactions(block, stxos) + if err != nil { + return err } } @@ -834,18 +1037,23 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags // to the main chain can be connected without violating any rules and // without actually connecting the block. // - // NOTE: bitcoind does these checks directly when it connects a block. - // The downside to that approach is that if any of these checks fail - // after disconnecting some blocks or attaching others, all of the + // NOTE: These checks could be done directly when connecting a block, + // however the downside to that approach is that if any of these checks + // fail after disconnecting some blocks or attaching others, all of the // operations have to be rolled back to get the chain back into the // state it was before the rule violation (or other failure). There are // at least a couple of ways accomplish that rollback, but both involve - // tweaking the chain. This approach catches these issues before ever - // modifying the chain. + // tweaking the chain and/or database. This approach catches these + // issues before ever modifying the chain. for e := attachNodes.Front(); e != nil; e = e.Next() { n := e.Value.(*blockNode) block := b.blockCache[*n.hash] - err := b.checkConnectBlock(n, block) + + // Notice the spent txout details are not requested here and + // thus will not be generated. This is done because the state + // is not being immediately written to the database, so it is + // not needed. + err := b.checkConnectBlock(n, block, view, nil) if err != nil { return err } @@ -857,14 +1065,35 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags return nil } + // Reset the view for the actual connection code below. This is + // required because the view was previously modified when checking if + // the reorg would be successful and the connection code requires the + // view to be valid from the viewpoint of each block being connected or + // disconnected. + view = NewUtxoViewpoint() + view.SetBestHash(b.bestNode.hash) + // Disconnect blocks from the main chain. - for e := detachNodes.Front(); e != nil; e = e.Next() { + for i, e := 0, detachNodes.Front(); e != nil; i, e = i+1, e.Next() { n := e.Value.(*blockNode) - block, err := b.db.FetchBlockBySha(n.hash) + block := detachBlocks[i] + + // Load all of the utxos referenced by the block that aren't + // already in the view. + err := view.fetchInputUtxos(b.db, block) if err != nil { return err } - err = b.disconnectBlock(n, block) + + // Update the view to unspend all of the spent txos and remove + // the utxos created by the block. + err = view.disconnectTransactions(block, detachSpentTxOuts[i]) + if err != nil { + return err + } + + // Update the database and chain state. + err = b.disconnectBlock(n, block, view) if err != nil { return err } @@ -874,7 +1103,26 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags for e := attachNodes.Front(); e != nil; e = e.Next() { n := e.Value.(*blockNode) block := b.blockCache[*n.hash] - err := b.connectBlock(n, block) + + // Load all of the utxos referenced by the block that aren't + // already in the view. + err := view.fetchInputUtxos(b.db, block) + if err != nil { + return err + } + + // Update the view to mark all utxos referenced by the block + // as spent and add all transactions being created by this block + // to it. Also, provide an stxo slice so the spent txout + // details are generated. + stxos := make([]spentTxOut, 0, countSpentOutputs(block)) + err = view.connectTransactions(block, &stxos) + if err != nil { + return err + } + + // Update the database and chain state. + err = b.connectBlock(n, block, view, stxos) if err != nil { return err } @@ -905,25 +1153,28 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List, flags // cumulatively has the most proof of work. // // The flags modify the behavior of this function as follows: -// - BFFastAdd: Avoids the call to checkConnectBlock which does several -// expensive transaction validation operations. +// - BFFastAdd: Avoids several expensive transaction validation operations. +// This is useful when using checkpoints. // - BFDryRun: Prevents the block from being connected and avoids modifying the // state of the memory chain index. Also, any log messages related to // modifying the state are avoided. +// +// This function MUST be called with the chain state lock held (for writes). func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, flags BehaviorFlags) error { fastAdd := flags&BFFastAdd == BFFastAdd dryRun := flags&BFDryRun == BFDryRun - // We haven't selected a best chain yet or we are extending the main - // (best) chain with a new block. This is the most common case. - if b.bestChain == nil || node.parent.hash.IsEqual(b.bestChain.hash) { + // We are extending the main (best) chain with a new block. This is the + // most common case. + if node.parentHash.IsEqual(b.bestNode.hash) { // Perform several checks to verify the block can be connected - // to the main chain (including whatever reorganization might - // be necessary to get this node to the main chain) without - // violating any rules and without actually connecting the - // block. + // to the main chain without violating any rules and without + // actually connecting the block. + view := NewUtxoViewpoint() + view.SetBestHash(node.parentHash) + stxos := make([]spentTxOut, 0, countSpentOutputs(block)) if !fastAdd { - err := b.checkConnectBlock(node, block) + err := b.checkConnectBlock(node, block, view, &stxos) if err != nil { return err } @@ -934,8 +1185,23 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla return nil } + // In the fast add case the code to check the block connection + // was skipped, so the utxo view needs to load the referenced + // utxos, spend them, and add the new utxos being created by + // this block. + if fastAdd { + err := view.fetchInputUtxos(b.db, block) + if err != nil { + return err + } + err = view.connectTransactions(block, &stxos) + if err != nil { + return err + } + } + // Connect the block to the main chain. - err := b.connectBlock(node, block) + err := b.connectBlock(node, block, view, stxos) if err != nil { return err } @@ -981,7 +1247,7 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla // We're extending (or creating) a side chain, but the cumulative // work for this new side chain is not enough to make it the new chain. - if node.workSum.Cmp(b.bestChain.workSum) <= 0 { + if node.workSum.Cmp(b.bestNode.workSum) <= 0 { // Skip Logging info when the dry run flag is set. if dryRun { return nil @@ -1037,24 +1303,22 @@ func (b *BlockChain) connectBestChain(node *blockNode, block *btcutil.Block, fla // - Latest block height is after the latest checkpoint (if enabled) // - Latest block has a timestamp newer than 24 hours ago // -// This function is NOT safe for concurrent access. +// This function is safe for concurrent access. func (b *BlockChain) IsCurrent(timeSource MedianTimeSource) bool { - // Not current if there isn't a main (best) chain yet. - if b.bestChain == nil { - return false - } + b.chainLock.RLock() + defer b.chainLock.RUnlock() // Not current if the latest main (best) chain height is before the // latest known good checkpoint (when checkpoints are enabled). - checkpoint := b.LatestCheckpoint() - if checkpoint != nil && b.bestChain.height < checkpoint.Height { + checkpoint := b.latestCheckpoint() + if checkpoint != nil && b.bestNode.height < checkpoint.Height { return false } // Not current if the latest best block has a timestamp before 24 hours // ago. minus24Hours := timeSource.AdjustedTime().Add(-24 * time.Hour) - if b.bestChain.timestamp.Before(minus24Hours) { + if b.bestNode.timestamp.Before(minus24Hours) { return false } @@ -1063,14 +1327,63 @@ func (b *BlockChain) IsCurrent(timeSource MedianTimeSource) bool { return true } -// New returns a BlockChain instance for the passed bitcoin network using the -// provided backing database. It accepts a callback on which notifications -// will be sent when various events take place. See the documentation for -// Notification and NotificationType for details on the types and contents of -// notifications. The provided callback can be nil if the caller is not -// interested in receiving notifications. -func New(db database.Db, params *chaincfg.Params, c NotificationCallback, sigCache *txscript.SigCache) *BlockChain { +// BestSnapshot returns information about the current best chain block and +// related state as of the current point in time. The returned instance must be +// treated as immutable since it is shared by all callers. +// +// This function is safe for concurrent access. +func (b *BlockChain) BestSnapshot() *BestState { + b.stateLock.RLock() + snapshot := b.stateSnapshot + b.stateLock.RUnlock() + return snapshot +} + +// Config is a descriptor which specifies the blockchain instance configuration. +type Config struct { + // DB defines the database which houses the blocks and will be used to + // store all metadata created by this package such as the utxo set. + // + // This field is required. + DB database.DB + + // ChainParams identifies which chain parameters the chain is associated + // with. + // + // This field is required. + ChainParams *chaincfg.Params + + // Notifications defines a callback to which notifications will be sent + // when various events take place. See the documentation for + // Notification and NotificationType for details on the types and + // contents of notifications. + // + // This field can be nil if the caller is not interested in receiving + // notifications. + Notifications NotificationCallback + + // SigCache defines a signature cache to use when when validating + // signatures. This is typically most useful when individual + // transactions are already being validated prior to their inclusion in + // a block such as what is usually done via a transaction memory pool. + // + // This field can be nil if the caller is not interested in using a + // signature cache. + SigCache *txscript.SigCache +} + +// New returns a BlockChain instance using the provided configuration details. +func New(config *Config) (*BlockChain, error) { + // Enforce required config fields. + if config.DB == nil { + return nil, AssertError("blockchain.New database is nil") + } + if config.ChainParams == nil { + return nil, AssertError("blockchain.New chain parameters nil") + } + // Generate a checkpoint by height map from the provided checkpoints. + params := config.ChainParams var checkpointsByHeight map[int32]*chaincfg.Checkpoint if len(params.Checkpoints) > 0 { checkpointsByHeight = make(map[int32]*chaincfg.Checkpoint) @@ -1081,18 +1394,30 @@ func New(db database.Db, params *chaincfg.Params, c NotificationCallback, sigCac } b := BlockChain{ - db: db, - sigCache: sigCache, - chainParams: params, checkpointsByHeight: checkpointsByHeight, - notifications: c, + db: config.DB, + chainParams: params, + notifications: config.Notifications, + sigCache: config.SigCache, root: nil, - bestChain: nil, + bestNode: nil, index: make(map[wire.ShaHash]*blockNode), depNodes: make(map[wire.ShaHash][]*blockNode), orphans: make(map[wire.ShaHash]*orphanBlock), prevOrphans: make(map[wire.ShaHash][]*orphanBlock), blockCache: make(map[wire.ShaHash]*btcutil.Block), } - return &b + + // Initialize the chain state from the passed database. When the db + // does not yet contain any chain state, both it and the chain state + // will be initialized to contain only the genesis block. + if err := b.initChainState(); err != nil { + return nil, err + } + + log.Infof("Chain state (height %d, hash %v, totaltx %d, work %v)", + b.bestNode.height, b.bestNode.hash, b.stateSnapshot.TotalTxns, + b.bestNode.workSum) + + return &b, nil } diff --git a/blockchain/chainio.go b/blockchain/chainio.go new file mode 100644 index 00000000..02869366 --- /dev/null +++ b/blockchain/chainio.go @@ -0,0 +1,1401 @@ +// Copyright (c) 2015-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/big" + "sort" + + database "github.com/btcsuite/btcd/database2" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" +) + +const ( + maxUint32 = 1<<32 - 1 +) + +var ( + // hashIndexBucketName is the name of the db bucket used to house to the + // block hash -> block height index. + hashIndexBucketName = []byte("hashidx") + + // heightIndexBucketName is the name of the db bucket used to house to + // the block height -> block hash index. + heightIndexBucketName = []byte("heightidx") + + // chainStateKeyName is the name of the db key used to store the best + // chain state. + chainStateKeyName = []byte("chainstate") + + // spendJournalBucketName is the name of the db bucket used to house + // transactions outputs that are spent in each block. + spendJournalBucketName = []byte("spendjournal") + + // utxoSetBucketName is the name of the db bucket used to house the + // unspent transaction output set. + utxoSetBucketName = []byte("utxoset") + + // byteOrder is the preferred byte order used for serializing numeric + // fields for storage in the database. + byteOrder = binary.LittleEndian +) + +// errNotInMainChain signifies that a block hash or height that is not in the +// main chain was requested. +type errNotInMainChain string + +// Error implements the error interface. +func (e errNotInMainChain) Error() string { + return string(e) +} + +// isNotInMainChainErr returns whether or not the passed error is an +// errNotInMainChain error. +func isNotInMainChainErr(err error) bool { + _, ok := err.(errNotInMainChain) + return ok +} + +// errDeserialize signifies that a problem was encountered when deserializing +// data. +type errDeserialize string + +// Error implements the error interface. +func (e errDeserialize) Error() string { + return string(e) +} + +// isDeserializeErr returns whether or not the passed error is an errDeserialize +// error. +func isDeserializeErr(err error) bool { + _, ok := err.(errDeserialize) + return ok +} + +// ----------------------------------------------------------------------------- +// The transaction spend journal consists of an entry for each block connected +// to the main chain which contains the transaction outputs the block spends +// serialized such that the order is the reverse of the order they were spent. +// +// This is required because reorganizing the chain necessarily entails +// disconnecting blocks to get back to the point of the fork which implies +// unspending all of the transaction outputs that each block previously spent. +// Since the utxo set, by definition, only contains unspent transaction outputs, +// the spent transaction outputs must be resurrected from somewhere. There is +// more than one way this could be done, however this is the most straight +// forward method that does not require having a transaction index and unpruned +// blockchain. +// +// NOTE: This format is NOT self describing. The additional details such as +// the number of entries (transaction inputs) are expected to come from the +// block itself and the utxo set. The rationale in doing this is to save a +// significant amount of space. This is also the reason the spent outputs are +// serialized in the reverse order they are spent because later transactions +// are allowed to spend outputs from earlier ones in the same block. +// +// The serialized format is: +// +// [
],... +// +// Field Type Size +// header code VLQ variable +// version VLQ variable +// compressed txout +// compressed amount VLQ variable +// compressed script []byte variable +// +// The serialized header code format is: +// bit 0 - containing transaction is a coinbase +// bits 1-x - height of the block that contains the spent txout +// +// NOTE: The header code and version are only encoded when the spent txout was +// the final unspent output of the containing transaction. Otherwise, the +// header code will be 0 and the version is not serialized at all. This is +// done because that information is only needed when the utxo set no longer +// has it. +// +// Example 1: +// From block 170 in main blockchain. +// +// 1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c +// <><><------------------------------------------------------------------> +// | | | +// | version compressed txout +// header code +// +// - header code: 0x13 (coinbase, height 9) +// - transaction version: 1 +// - compressed txout 0: +// - 0x32: VLQ-encoded compressed amount for 5000000000 (50 BTC) +// - 0x05: special script type pay-to-pubkey +// - 0x11...5c: x-coordinate of the pubkey +// +// Example 2: +// Adapted from block 100025 in main blockchain. +// +// 0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e868b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec +// <><----------------------------------------------><----><><----------------------------------------------> +// | | | | | +// | compressed txout | version compressed txout +// header code header code +// +// - Last spent output: +// - header code: 0x00 (was not the final unspent output for containing tx) +// - transaction version: Nothing since header code is 0 +// - compressed txout: +// - 0x91f20f: VLQ-encoded compressed amount for 34405000000 (344.05 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0x6e...86: pubkey hash +// - Second to last spent output: +// - header code: 0x8b9970 (not coinbase, height 100024) +// - transaction version: 1 +// - compressed txout: +// - 0x86c647: VLQ-encoded compressed amount for 13761000000 (137.61 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0xb2...ec: pubkey hash +// ----------------------------------------------------------------------------- + +// spentTxOut contains a spent transaction output and potentially additional +// contextual information such as whether or not it was contained in a coinbase +// transaction, the version of the transaction it was contained in, and which +// block height the containing transaction was included in. As described in +// the comments above, the additional contextual information will only be valid +// when this spent txout is spending the last unspent output of the containing +// transaction. +type spentTxOut struct { + compressed bool // The amount and public key script are compressed. + version int32 // The version of creating tx. + amount int64 // The amount of the output. + pkScript []byte // The public key script for the output. + + // These fields are only set when this is spending the final output of + // the creating tx. + height int32 // Height of the the block containing the creating tx. + isCoinBase bool // Whether creating tx is a coinbase. +} + +// spentTxOutHeaderCode returns the calculated header code to be used when +// serializing the provided stxo entry. +func spentTxOutHeaderCode(stxo *spentTxOut) uint64 { + // The header code is 0 when there is no height set for the stxo. + if stxo.height == 0 { + return 0 + } + + // As described in the serialization format comments, the header code + // encodes the height shifted over one bit and the coinbase flag in the + // lowest bit. + headerCode := uint64(stxo.height) << 1 + if stxo.isCoinBase { + headerCode |= 0x01 + } + + return headerCode +} + +// spentTxOutSerializeSize returns the number of bytes it would take to +// serialize the passed stxo according to the format described above. +func spentTxOutSerializeSize(stxo *spentTxOut) int { + headerCode := spentTxOutHeaderCode(stxo) + size := serializeSizeVLQ(headerCode) + if headerCode != 0 { + size += serializeSizeVLQ(uint64(stxo.version)) + } + return size + compressedTxOutSize(uint64(stxo.amount), stxo.pkScript, + stxo.version, stxo.compressed) +} + +// putSpentTxOut serializes the passed stxo according to the format described +// above directly into the passed target byte slice. The target byte slice must +// be at least large enough to handle the number of bytes returned by the +// spentTxOutSerializeSize function or it will panic. +func putSpentTxOut(target []byte, stxo *spentTxOut) int { + headerCode := spentTxOutHeaderCode(stxo) + offset := putVLQ(target, headerCode) + if headerCode != 0 { + offset += putVLQ(target[offset:], uint64(stxo.version)) + } + return offset + putCompressedTxOut(target[offset:], uint64(stxo.amount), + stxo.pkScript, stxo.version, stxo.compressed) +} + +// decodeSpentTxOut decodes the passed serialized stxo entry, possibly followed +// by other data, into the passed stxo struct. It returns the number of bytes +// read. +// +// Since the serialized stxo entry does not contain the height, version, or +// coinbase flag of the containing transaction when it still has utxos, the +// caller is responsible for passing in the containing transaction version in +// that case. The provided version is ignore when it is serialized as a part of +// the stxo. +// +// An error will be returned if the version is not serialized as a part of the +// stxo and is also not provided to the function. +func decodeSpentTxOut(serialized []byte, stxo *spentTxOut, txVersion int32) (int, error) { + // Ensure there are bytes to decode. + if len(serialized) == 0 { + return 0, errDeserialize("no serialized bytes") + } + + // Deserialize the header code. + code, offset := deserializeVLQ(serialized) + if offset >= len(serialized) { + return offset, errDeserialize("unexpected end of data after " + + "header code") + } + + // Decode the header code and deserialize the containing transaction + // version if needed. + // + // Bit 0 indicates containing transaction is a coinbase. + // Bits 1-x encode height of containing transaction. + if code != 0 { + version, bytesRead := deserializeVLQ(serialized[offset:]) + offset += bytesRead + if offset >= len(serialized) { + return offset, errDeserialize("unexpected end of data " + + "after version") + } + + stxo.isCoinBase = code&0x01 != 0 + stxo.height = int32(code >> 1) + stxo.version = int32(version) + } else { + // Ensure a tx version was specified if the stxo did not encode + // it. This should never happen unless there is database + // corruption or this function is being called without the + // proper state. + if txVersion == 0 { + return offset, AssertError("decodeSpentTxOut called " + + "without a containing tx version when the " + + "serialized stxo that does not encode the " + + "version") + } + stxo.version = txVersion + } + + // Decode the compressed txout. + compAmount, compScript, bytesRead, err := decodeCompressedTxOut( + serialized[offset:], stxo.version) + offset += bytesRead + if err != nil { + return offset, errDeserialize(fmt.Sprintf("unable to decode "+ + "txout: %v", err)) + } + stxo.amount = int64(compAmount) + stxo.pkScript = compScript + stxo.compressed = true + return offset, nil +} + +// deserializeSpendJournalEntry decodes the passed serialized byte slice into a +// slice of spent txouts according to the format described in detail above. +// +// Since the serialization format is not self describing, as noted in the +// format comments, this function also requires the transactions that spend the +// txouts and a utxo view that contains any remaining existing utxos in the +// transactions referenced by the inputs to the passed transasctions. +func deserializeSpendJournalEntry(serialized []byte, txns []*wire.MsgTx, view *UtxoViewpoint) ([]spentTxOut, error) { + // Calculate the total number of stxos. + var numStxos int + for _, tx := range txns { + numStxos += len(tx.TxIn) + } + + // When a block has no spent txouts there is nothing to serialize. + if len(serialized) == 0 { + // Ensure the block actually has no stxos. This should never + // happen unless there is database corruption or an empty entry + // erroneously made its way into the database. + if numStxos != 0 { + return nil, AssertError(fmt.Sprintf("mismatched spend "+ + "journal serialization - no serialization for "+ + "expected %d stxos", numStxos)) + } + + return nil, nil + } + + // Loop backwards through all transactions so everything is read in + // reverse order to match the serialization order. + stxoIdx := numStxos - 1 + stxoInFlight := make(map[wire.ShaHash]int) + offset := 0 + stxos := make([]spentTxOut, numStxos) + for txIdx := len(txns) - 1; txIdx > -1; txIdx-- { + tx := txns[txIdx] + + // Loop backwards through all of the transaction inputs and read + // the associated stxo. + for txInIdx := len(tx.TxIn) - 1; txInIdx > -1; txInIdx-- { + txIn := tx.TxIn[txInIdx] + stxo := &stxos[stxoIdx] + stxoIdx-- + + // Get the transaction version for the stxo based on + // whether or not it should be serialized as a part of + // the stxo. Recall that it is only serialized when the + // stxo spends the final utxo of a transaction. Since + // they are deserialized in reverse order, this means + // the first time an entry for a given containing tx is + // encountered that is not already in the utxo view it + // must have been the final spend and thus the extra + // data will be serialized with the stxo. Otherwise, + // the version must be pulled from the utxo entry. + // + // Since the view is not actually modified as the stxos + // are read here and it's possible later entries + // reference earlier ones, an inflight map is maintained + // to detect this case and pull the tx version from the + // entry that contains the version information as just + // described. + var txVersion int32 + originHash := &txIn.PreviousOutPoint.Hash + entry := view.LookupEntry(originHash) + if entry != nil { + txVersion = entry.Version() + } else if idx, ok := stxoInFlight[*originHash]; ok { + txVersion = stxos[idx].version + } else { + stxoInFlight[*originHash] = stxoIdx + 1 + } + + n, err := decodeSpentTxOut(serialized[offset:], stxo, + txVersion) + offset += n + if err != nil { + return nil, errDeserialize(fmt.Sprintf("unable "+ + "to decode stxo for %v: %v", + txIn.PreviousOutPoint, err)) + } + } + } + + return stxos, nil +} + +// serializeSpendJournalEntry serializes all of the passed spent txouts into a +// single byte slice according to the format described in detail above. +func serializeSpendJournalEntry(stxos []spentTxOut) []byte { + if len(stxos) == 0 { + return nil + } + + // Calculate the size needed to serialize the entire journal entry. + var size int + for i := range stxos { + size += spentTxOutSerializeSize(&stxos[i]) + } + serialized := make([]byte, size) + + // Serialize each individual stxo directly into the slice in reverse + // order one after the other. + var offset int + for i := len(stxos) - 1; i > -1; i-- { + offset += putSpentTxOut(serialized[offset:], &stxos[i]) + } + + return serialized +} + +// dbFetchSpendJournalEntry fetches the spend journal entry for the passed +// block and deserializes it into a slice of spent txout entries. The provided +// view MUST have the utxos referenced by all of the transactions available for +// the passed block since that information is required to reconstruct the spent +// txouts. +func dbFetchSpendJournalEntry(dbTx database.Tx, block *btcutil.Block, view *UtxoViewpoint) ([]spentTxOut, error) { + // Exclude the coinbase transaction since it can't spend anything. + spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName) + serialized := spendBucket.Get(block.Sha()[:]) + blockTxns := block.MsgBlock().Transactions[1:] + stxos, err := deserializeSpendJournalEntry(serialized, blockTxns, view) + if err != nil { + // Ensure any deserialization errors are returned as database + // corruption errors. + if isDeserializeErr(err) { + return nil, database.Error{ + ErrorCode: database.ErrCorruption, + Description: fmt.Sprintf("corrupt spend "+ + "information for %v: %v", block.Sha(), + err), + } + } + + return nil, err + } + + return stxos, nil +} + +// dbPutSpendJournalEntry uses an existing database transaction to update the +// spend journal entry for the given block hash using the provided slice of +// spent txouts. The spent txouts slice must contain an entry for every txout +// the transactions in the block spend in the order they are spent. +func dbPutSpendJournalEntry(dbTx database.Tx, blockHash *wire.ShaHash, stxos []spentTxOut) error { + spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName) + serialized := serializeSpendJournalEntry(stxos) + return spendBucket.Put(blockHash[:], serialized) +} + +// dbRemoveSpendJournalEntry uses an existing database transaction to remove the +// spend journal entry for the passed block hash. +func dbRemoveSpendJournalEntry(dbTx database.Tx, blockHash *wire.ShaHash) error { + spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName) + return spendBucket.Delete(blockHash[:]) +} + +// ----------------------------------------------------------------------------- +// The unspent transaction output (utxo) set consists of an entry for each +// transaction which contains a utxo serialized using a format that is highly +// optimized to reduce space using domain specific compression algorithms. This +// format is a slightly modified version of the format used in Bitcoin Core. +// +// The serialized format is: +// +//
[,...] +// +// Field Type Size +// version VLQ variable +// block height VLQ variable +// header code VLQ variable +// unspentness bitmap []byte variable +// compressed txouts +// compressed amount VLQ variable +// compressed script []byte variable +// +// The serialized header code format is: +// bit 0 - containing transaction is a coinbase +// bit 1 - output zero is unspent +// bit 2 - output one is unspent +// bits 3-x - number of bytes in unspentness bitmap. When both bits 1 and 2 +// are unset, it encodes N-1 since there must be at least one unspent +// output. +// +// The rationale for the header code scheme is as follows: +// - Transactions which only pay to a single output and a change output are +// extremely common, thus an extra byte for the unspentness bitmap can be +// avoided for them by encoding those two outputs in the low order bits. +// - Given it is encoded as a VLQ which can encode values up to 127 with a +// single byte, that leaves 4 bits to represent the number of bytes in the +// unspentness bitmap while still only consuming a single byte for the +// header code. In other words, an unspentness bitmap with up to 120 +// transaction outputs can be encoded with a single-byte header code. +// This covers the vast majority of transactions. +// - Encoding N-1 bytes when both bits 1 and 2 are unset allows an additional +// 8 outpoints to be encoded before causing the header code to require an +// additional byte. +// +// Example 1: +// From tx in main blockchain: +// Blk 1, 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098 +// +// 010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52 +// <><><><------------------------------------------------------------------> +// | | \--------\ | +// | height | compressed txout 0 +// version header code +// +// - version: 1 +// - height: 1 +// - header code: 0x03 (coinbase, output zero unspent, 0 bytes of unspentness) +// - unspentness: Nothing since it is zero bytes +// - compressed txout 0: +// - 0x32: VLQ-encoded compressed amount for 5000000000 (50 BTC) +// - 0x04: special script type pay-to-pubkey +// - 0x96...52: x-coordinate of the pubkey +// +// Example 2: +// From tx in main blockchain: +// Blk 113931, 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f +// +// 0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58 +// <><----><><><------------------------------------------><--------------------------------------------> +// | | | \-------------------\ | | +// version | \--------\ unspentness | compressed txout 2 +// height header code compressed txout 0 +// +// - version: 1 +// - height: 113931 +// - header code: 0x0a (output zero unspent, 1 byte in unspentness bitmap) +// - unspentness: [0x01] (bit 0 is set, so output 0+2 = 2 is unspent) +// NOTE: It's +2 since the first two outputs are encoded in the header code +// - compressed txout 0: +// - 0x12: VLQ-encoded compressed amount for 20000000 (0.2 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0xe2...8a: pubkey hash +// - compressed txout 2: +// - 0x8009: VLQ-encoded compressed amount for 15000000 (0.15 BTC) +// - 0x00: special script type pay-to-pubkey-hash +// - 0xb8...58: pubkey hash +// +// Example 3: +// From tx in main blockchain: +// Blk 338156, 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620 +// +// 0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6 +// <><----><><----><--------------------------------------------------> +// | | | \-----------------\ | +// version | \--------\ unspentness | +// height header code compressed txout 22 +// +// - version: 1 +// - height: 338156 +// - header code: 0x10 (2+1 = 3 bytes in unspentness bitmap) +// NOTE: It's +1 since neither bit 1 nor 2 are set, so N-1 is encoded. +// - unspentness: [0x00 0x00 0x10] (bit 20 is set, so output 20+2 = 22 is unspent) +// NOTE: It's +2 since the first two outputs are encoded in the header code +// - compressed txout 22: +// - 0x8ba5b9e763: VLQ-encoded compressed amount for 366875659 (3.66875659 BTC) +// - 0x01: special script type pay-to-script-hash +// - 0x1d...e6: script hash +// ----------------------------------------------------------------------------- + +// utxoEntryHeaderCode returns the calculated header code to be used when +// serializing the provided utxo entry and the number of bytes needed to encode +// the unspentness bitmap. +func utxoEntryHeaderCode(entry *UtxoEntry, highestOutputIndex uint32) (uint64, int, error) { + // The first two outputs are encoded separately, so offset the index + // accordingly to calculate the correct number of bytes needed to encode + // up to the highest unspent output index. + numBitmapBytes := int((highestOutputIndex + 6) / 8) + + // As previously described, one less than the number of bytes is encoded + // when both output 0 and 1 are spent because there must be at least one + // unspent output. Adjust the number of bytes to encode accordingly and + // encode the value by shifting it over 3 bits. + output0Unspent := !entry.IsOutputSpent(0) + output1Unspent := !entry.IsOutputSpent(1) + var numBitmapBytesAdjustment int + if !output0Unspent && !output1Unspent { + if numBitmapBytes == 0 { + return 0, 0, AssertError("attempt to serialize utxo " + + "header for fully spent transaction") + } + numBitmapBytesAdjustment = 1 + } + headerCode := uint64(numBitmapBytes-numBitmapBytesAdjustment) << 3 + + // Set the coinbase, output 0, and output 1 bits in the header code + // accordingly. + if entry.isCoinBase { + headerCode |= 0x01 // bit 0 + } + if output0Unspent { + headerCode |= 0x02 // bit 1 + } + if output1Unspent { + headerCode |= 0x04 // bit 2 + } + + return headerCode, numBitmapBytes, nil +} + +// serializeUtxoEntry returns the entry serialized to a format that is suitable +// for long-term storage. The format is described in detail above. +func serializeUtxoEntry(entry *UtxoEntry) ([]byte, error) { + // Fully spent entries have no serialization. + if entry.IsFullySpent() { + return nil, nil + } + + // Determine the output order by sorting the sparse output index keys. + outputOrder := make([]int, 0, len(entry.sparseOutputs)) + for outputIndex := range entry.sparseOutputs { + outputOrder = append(outputOrder, int(outputIndex)) + } + sort.Ints(outputOrder) + + // Encode the header code and determine the number of bytes the + // unspentness bitmap needs. + highIndex := uint32(outputOrder[len(outputOrder)-1]) + headerCode, numBitmapBytes, err := utxoEntryHeaderCode(entry, highIndex) + if err != nil { + return nil, err + } + + // The number of bitmap bytes must not exceed a max uint32. This could + // only happen if the number of txouts in a transaction were to exceed + // (2^32 - 1)*8 == (2^35 - 8). This should never happen, but assert + // the condition because if it ever becomes true some distant time in + // the future, the code must be changed. + if numBitmapBytes > maxUint32 { + return nil, AssertError("unspentness bitmap too large to " + + "serialize") + } + + // Calculate the size needed to serialize the entry. + size := serializeSizeVLQ(uint64(entry.version)) + + serializeSizeVLQ(uint64(entry.blockHeight)) + + serializeSizeVLQ(headerCode) + numBitmapBytes + for _, outputIndex := range outputOrder { + out := entry.sparseOutputs[uint32(outputIndex)] + if out.spent { + continue + } + size += compressedTxOutSize(uint64(out.amount), out.pkScript, + entry.version, out.compressed) + } + + // Serialize the version, block height of the containing transaction, + // and header code. + serialized := make([]byte, size) + offset := putVLQ(serialized, uint64(entry.version)) + offset += putVLQ(serialized[offset:], uint64(entry.blockHeight)) + offset += putVLQ(serialized[offset:], headerCode) + + // Serialize the unspentness bitmap. + for i := uint32(0); i < uint32(numBitmapBytes); i++ { + unspentBits := byte(0) + for j := uint32(0); j < 8; j++ { + // The first 2 outputs are encoded via the header code, + // so adjust the output index accordingly. + if !entry.IsOutputSpent(2 + i*8 + j) { + unspentBits |= 1 << uint8(j) + } + } + serialized[offset] = unspentBits + offset++ + } + + // Serialize the compressed unspent transaction outputs. Outputs that + // are already compressed are serialized without modifications. + for _, outputIndex := range outputOrder { + out := entry.sparseOutputs[uint32(outputIndex)] + if out.spent { + continue + } + + offset += putCompressedTxOut(serialized[offset:], + uint64(out.amount), out.pkScript, entry.version, + out.compressed) + } + + return serialized, nil +} + +// deserializeUtxoEntry decodes a utxo entry from the passed serialized byte +// slice into a new UtxoEntry using a format that is suitable for long-term +// storage. The format is described in detail above. +func deserializeUtxoEntry(serialized []byte) (*UtxoEntry, error) { + // Deserialize the version. + version, bytesRead := deserializeVLQ(serialized) + offset := bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after version") + } + + // Deserialize the block height. + blockHeight, bytesRead := deserializeVLQ(serialized[offset:]) + offset += bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after height") + } + + // Deserialize the header code. + code, bytesRead := deserializeVLQ(serialized[offset:]) + offset += bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after header") + } + + // Decode the header code. + // + // Bit 0 indicates whether the containing transaction is a coinbase. + // Bit 1 indicates output 0 is unspent. + // Bit 2 indicates output 1 is unspent. + // Bits 3-x encodes the number of non-zero unspentness bitmap bytes that + // follow. When both output 0 and 1 are spent, it encodes N-1. + isCoinBase := code&0x01 != 0 + output0Unspent := code&0x02 != 0 + output1Unspent := code&0x04 != 0 + numBitmapBytes := code >> 3 + if !output0Unspent && !output1Unspent { + numBitmapBytes++ + } + + // Ensure there are enough bytes left to deserialize the unspentness + // bitmap. + if uint64(len(serialized[offset:])) < numBitmapBytes { + return nil, errDeserialize("unexpected end of data for " + + "unspentness bitmap") + } + + // Create a new utxo entry with the details deserialized above to house + // all of the utxos. + entry := newUtxoEntry(int32(version), isCoinBase, int32(blockHeight)) + + // Add sparse output for unspent outputs 0 and 1 as needed based on the + // details provided by the header code. + var outputIndexes []uint32 + if output0Unspent { + outputIndexes = append(outputIndexes, 0) + } + if output1Unspent { + outputIndexes = append(outputIndexes, 1) + } + + // Decode the unspentness bitmap adding a sparse output for each unspent + // output. + for i := uint32(0); i < uint32(numBitmapBytes); i++ { + unspentBits := serialized[offset] + for j := uint32(0); j < 8; j++ { + if unspentBits&0x01 != 0 { + // The first 2 outputs are encoded via the + // header code, so adjust the output number + // accordingly. + outputNum := 2 + i*8 + j + outputIndexes = append(outputIndexes, outputNum) + } + unspentBits >>= 1 + } + offset++ + } + + // Decode and add all of the utxos. + for i, outputIndex := range outputIndexes { + // Decode the next utxo. The script and amount fields of the + // utxo output are left compressed so decompression can be + // avoided on those that are not accessed. This is done since + // it is quite common for a redeeming transaction to only + // reference a single utxo from a referenced transaction. + compAmount, compScript, bytesRead, err := decodeCompressedTxOut( + serialized[offset:], int32(version)) + if err != nil { + return nil, errDeserialize(fmt.Sprintf("unable to "+ + "decode utxo at index %d: %v", i, err)) + } + offset += bytesRead + + entry.sparseOutputs[outputIndex] = &utxoOutput{ + spent: false, + compressed: true, + pkScript: compScript, + amount: int64(compAmount), + } + } + + return entry, nil +} + +// dbFetchUtxoEntry uses an existing database transaction to fetch all unspent +// outputs for the provided Bitcoin transaction hash from the utxo set. +// +// When there is no entry for the provided hash, nil will be returned for the +// both the entry and the error. +func dbFetchUtxoEntry(dbTx database.Tx, hash *wire.ShaHash) (*UtxoEntry, error) { + // Fetch the unspent transaction output information for the passed + // transaction hash. Return now when there is no entry. + utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) + serializedUtxo := utxoBucket.Get(hash[:]) + if serializedUtxo == nil { + return nil, nil + } + + // A non-nil zero-length entry means there is an entry in the database + // for a fully spent transaction which should never be the case. + if len(serializedUtxo) == 0 { + return nil, AssertError(fmt.Sprintf("database contains entry "+ + "for fully spent tx %v", hash)) + } + + // Deserialize the utxo entry and return it. + entry, err := deserializeUtxoEntry(serializedUtxo) + if err != nil { + // Ensure any deserialization errors are returned as database + // corruption errors. + if isDeserializeErr(err) { + return nil, database.Error{ + ErrorCode: database.ErrCorruption, + Description: fmt.Sprintf("corrupt utxo entry "+ + "for %v: %v", hash, err), + } + } + + return nil, err + } + + return entry, nil +} + +// dbPutUtxoView uses an existing database transaction to update the utxo set +// in the database based on the provided utxo view contents and state. In +// particular, only the entries that have been marked as modified are written +// to the database. +func dbPutUtxoView(dbTx database.Tx, view *UtxoViewpoint) error { + utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) + for txHashIter, entry := range view.entries { + // No need to update the database if the entry was not modified. + if entry == nil || !entry.modified { + continue + } + + // Serialize the utxo entry without any entries that have been + // spent. + serialized, err := serializeUtxoEntry(entry) + if err != nil { + return err + } + + // Make a copy of the hash because the iterator changes on each + // loop iteration and thus slicing it directly would cause the + // data to change out from under the put/delete funcs below. + txHash := txHashIter + + // Remove the utxo entry if it is now fully spent. + if serialized == nil { + if err := utxoBucket.Delete(txHash[:]); err != nil { + return err + } + + continue + } + + // At this point the utxo entry is not fully spent, so store its + // serialization in the database. + err = utxoBucket.Put(txHash[:], serialized) + if err != nil { + return err + } + } + + return nil +} + +// ----------------------------------------------------------------------------- +// The block index consists of two buckets with an entry for every block in the +// main chain. One bucket is for the hash to height mapping and the other is +// for the height to hash mapping. +// +// The serialized format for values in the hash to height bucket is: +// +// +// Field Type Size +// height uint32 4 bytes +// +// The serialized format for values in the height to hash bucket is: +// +// +// Field Type Size +// hash wire.ShaHash wire.HashSize +// ----------------------------------------------------------------------------- + +// dbPutBlockIndex uses an existing database transaction to update or add the +// block index entries for the hash to height and height to hash mappings for +// the provided values. +func dbPutBlockIndex(dbTx database.Tx, hash *wire.ShaHash, height int32) error { + // Serialize the height for use in the index entries. + var serializedHeight [4]byte + byteOrder.PutUint32(serializedHeight[:], uint32(height)) + + // Add the block hash to height mapping to the index. + meta := dbTx.Metadata() + hashIndex := meta.Bucket(hashIndexBucketName) + if err := hashIndex.Put(hash[:], serializedHeight[:]); err != nil { + return err + } + + // Add the block height to hash mapping to the index. + heightIndex := meta.Bucket(heightIndexBucketName) + return heightIndex.Put(serializedHeight[:], hash[:]) +} + +// dbRemoveBlockIndex uses an existing database transaction remove block index +// entries from the hash to height and height to hash mappings for the provided +// values. +func dbRemoveBlockIndex(dbTx database.Tx, hash *wire.ShaHash, height int32) error { + // Remove the block hash to height mapping. + meta := dbTx.Metadata() + hashIndex := meta.Bucket(hashIndexBucketName) + if err := hashIndex.Delete(hash[:]); err != nil { + return err + } + + // Remove the block height to hash mapping. + var serializedHeight [4]byte + byteOrder.PutUint32(serializedHeight[:], uint32(height)) + heightIndex := meta.Bucket(heightIndexBucketName) + return heightIndex.Delete(serializedHeight[:]) +} + +// dbFetchHeightByHash uses an existing database transaction to retrieve the +// height for the provided hash from the index. +func dbFetchHeightByHash(dbTx database.Tx, hash *wire.ShaHash) (int32, error) { + meta := dbTx.Metadata() + hashIndex := meta.Bucket(hashIndexBucketName) + serializedHeight := hashIndex.Get(hash[:]) + if serializedHeight == nil { + str := fmt.Sprintf("block %s is not in the main chain", hash) + return 0, errNotInMainChain(str) + } + + return int32(byteOrder.Uint32(serializedHeight)), nil +} + +// dbFetchHashByHeight uses an existing database transaction to retrieve the +// hash for the provided height from the index. +func dbFetchHashByHeight(dbTx database.Tx, height int32) (*wire.ShaHash, error) { + var serializedHeight [4]byte + byteOrder.PutUint32(serializedHeight[:], uint32(height)) + + meta := dbTx.Metadata() + heightIndex := meta.Bucket(heightIndexBucketName) + hashBytes := heightIndex.Get(serializedHeight[:]) + if hashBytes == nil { + str := fmt.Sprintf("no block at height %d exists", height) + return nil, errNotInMainChain(str) + } + + var hash wire.ShaHash + copy(hash[:], hashBytes) + return &hash, nil +} + +// ----------------------------------------------------------------------------- +// The best chain state consists of the best block hash and height, the total +// number of transactions up to and including those in the best block, and the +// accumulated work sum up to and including the best block. +// +// The serialized format is: +// +// +// +// Field Type Size +// block hash wire.ShaHash wire.HashSize +// block height uint32 4 bytes +// total txns uint64 8 bytes +// work sum length uint32 4 bytes +// work sum big.Int work sum length +// ----------------------------------------------------------------------------- + +// bestChainState represents the data to be stored the database for the current +// best chain state. +type bestChainState struct { + hash wire.ShaHash + height uint32 + totalTxns uint64 + workSum *big.Int +} + +// serializeBestChainState returns the serialization of the passed block best +// chain state. This is data to be stored in the chain state bucket. +func serializeBestChainState(state bestChainState) []byte { + // Calculate the full size needed to serialize the chain state. + workSumBytes := state.workSum.Bytes() + workSumBytesLen := uint32(len(workSumBytes)) + serializedLen := wire.HashSize + 4 + 8 + 4 + workSumBytesLen + + // Serialize the chain state. + serializedData := make([]byte, serializedLen) + copy(serializedData[0:wire.HashSize], state.hash[:]) + offset := uint32(wire.HashSize) + byteOrder.PutUint32(serializedData[offset:], state.height) + offset += 4 + byteOrder.PutUint64(serializedData[offset:], state.totalTxns) + offset += 8 + byteOrder.PutUint32(serializedData[offset:], workSumBytesLen) + offset += 4 + copy(serializedData[offset:], workSumBytes) + return serializedData[:] +} + +// deserializeBestChainState deserializes the passed serialized best chain +// state. This is data stored in the chain state bucket and is updated after +// every block is connected or disconnected form the main chain. +// block. +func deserializeBestChainState(serializedData []byte) (bestChainState, error) { + // Ensure the serialized data has enough bytes to properly deserialize + // the hash, height, total transactions, and work sum length. + if len(serializedData) < wire.HashSize+16 { + return bestChainState{}, database.Error{ + ErrorCode: database.ErrCorruption, + Description: "corrupt best chain state", + } + } + + state := bestChainState{} + copy(state.hash[:], serializedData[0:wire.HashSize]) + offset := uint32(wire.HashSize) + state.height = byteOrder.Uint32(serializedData[offset : offset+4]) + offset += 4 + state.totalTxns = byteOrder.Uint64(serializedData[offset : offset+8]) + offset += 8 + workSumBytesLen := byteOrder.Uint32(serializedData[offset : offset+4]) + offset += 4 + + // Ensure the serialized data has enough bytes to deserialize the work + // sum. + if uint32(len(serializedData[offset:])) < workSumBytesLen { + return bestChainState{}, database.Error{ + ErrorCode: database.ErrCorruption, + Description: "corrupt best chain state", + } + } + workSumBytes := serializedData[offset : offset+workSumBytesLen] + state.workSum = new(big.Int).SetBytes(workSumBytes) + + return state, nil +} + +// dbPutBestState uses an existing database transaction to update the best chain +// state with the given parameters. +func dbPutBestState(dbTx database.Tx, snapshot *BestState, workSum *big.Int) error { + // Serialize the current best chain state. + serializedData := serializeBestChainState(bestChainState{ + hash: *snapshot.Hash, + height: uint32(snapshot.Height), + totalTxns: snapshot.TotalTxns, + workSum: workSum, + }) + + // Store the current best chain state into the database. + return dbTx.Metadata().Put(chainStateKeyName, serializedData) +} + +// createChainState initializes both the database and the chain state to the +// genesis block. This includes creating the necessary buckets and inserting +// the genesis block, so it must only be called on an uninitialized database. +func (b *BlockChain) createChainState() error { + // Create a new node from the genesis block and set it as both the root + // node and the best node. + genesisBlock := btcutil.NewBlock(b.chainParams.GenesisBlock) + header := &genesisBlock.MsgBlock().Header + node := newBlockNode(header, genesisBlock.Sha(), 0) + node.inMainChain = true + b.bestNode = node + b.root = node + + // Add the new node to the index which is used for faster lookups. + b.index[*node.hash] = node + + // Initialize the state related to the best block. + numTxns := uint64(len(genesisBlock.MsgBlock().Transactions)) + blockSize := uint64(genesisBlock.MsgBlock().SerializeSize()) + b.stateSnapshot = newBestState(b.bestNode, blockSize, numTxns, numTxns) + + // Create the initial the database chain state including creating the + // necessary index buckets and inserting the genesis block. + err := b.db.Update(func(dbTx database.Tx) error { + // Create the bucket that houses the chain block hash to height + // index. + meta := dbTx.Metadata() + _, err := meta.CreateBucket(hashIndexBucketName) + if err != nil { + return err + } + + // Create the bucket that houses the chain block height to hash + // index. + _, err = meta.CreateBucket(heightIndexBucketName) + if err != nil { + return err + } + + // Create the bucket that houses the spend journal data. + _, err = meta.CreateBucket(spendJournalBucketName) + if err != nil { + return err + } + + // Create the bucket that houses the utxo set. Note that the + // genesis block coinbase transaction is intentionally not + // inserted here since it is not spendable by consensus rules. + _, err = meta.CreateBucket(utxoSetBucketName) + if err != nil { + return err + } + + // Add the genesis block hash to height and height to hash + // mappings to the index. + err = dbPutBlockIndex(dbTx, b.bestNode.hash, b.bestNode.height) + if err != nil { + return err + } + + // Store the current best chain state into the database. + err = dbPutBestState(dbTx, b.stateSnapshot, b.bestNode.workSum) + if err != nil { + return err + } + + // Store the genesis block into the database. + return dbTx.StoreBlock(genesisBlock) + }) + return err +} + +// initChainState attempts to load and initialize the chain state from the +// database. When the db does not yet contain any chain state, both it and the +// chain state are initialized to the genesis block. +func (b *BlockChain) initChainState() error { + // Attempt to load the chain state from the database. + var isStateInitialized bool + err := b.db.View(func(dbTx database.Tx) error { + // Fetch the stored chain state from the database metadata. + // When it doesn't exist, it means the database hasn't been + // initialized for use with chain yet, so break out now to allow + // that to happen under a writable database transaction. + serializedData := dbTx.Metadata().Get(chainStateKeyName) + if serializedData == nil { + return nil + } + log.Tracef("Serialized chain state: %x", serializedData) + state, err := deserializeBestChainState(serializedData) + if err != nil { + return err + } + + // Load the raw block bytes for the best block. + blockBytes, err := dbTx.FetchBlock(&state.hash) + if err != nil { + return err + } + var block wire.MsgBlock + err = block.Deserialize(bytes.NewReader(blockBytes)) + if err != nil { + return err + } + + // Create a new node and set it as both the root node and the + // best node. The preceding nodes will be loaded on demand as + // needed. + header := &block.Header + node := newBlockNode(header, &state.hash, int32(state.height)) + node.inMainChain = true + node.workSum = state.workSum + b.bestNode = node + b.root = node + + // Add the new node to the indices for faster lookups. + prevHash := node.parentHash + b.index[*node.hash] = node + b.depNodes[*prevHash] = append(b.depNodes[*prevHash], node) + + // Initialize the state related to the best block. + blockSize := uint64(len(blockBytes)) + numTxns := uint64(len(block.Transactions)) + b.stateSnapshot = newBestState(b.bestNode, blockSize, numTxns, + state.totalTxns) + + isStateInitialized = true + return nil + }) + if err != nil { + return err + } + + // There is nothing more to do if the chain state was initialized. + if isStateInitialized { + return nil + } + + // At this point the database has not already been initialized, so + // initialize both it and the chain state to the genesis block. + return b.createChainState() +} + +// dbFetchHeaderByHash uses an existing database transaction to retrieve the +// block header for the provided hash. +func dbFetchHeaderByHash(dbTx database.Tx, hash *wire.ShaHash) (*wire.BlockHeader, error) { + headerBytes, err := dbTx.FetchBlockHeader(hash) + if err != nil { + return nil, err + } + + var header wire.BlockHeader + err = header.Deserialize(bytes.NewReader(headerBytes)) + if err != nil { + return nil, err + } + + return &header, nil +} + +// dbFetchHeaderByHeight uses an existing database transaction to retrieve the +// block header for the provided height. +func dbFetchHeaderByHeight(dbTx database.Tx, height int32) (*wire.BlockHeader, error) { + hash, err := dbFetchHashByHeight(dbTx, height) + if err != nil { + return nil, err + } + + return dbFetchHeaderByHash(dbTx, hash) +} + +// dbFetchBlockByHash uses an existing database transaction to retrieve the raw +// block for the provided hash, deserialize it, retrieve the appropriate height +// from the index, and return a btcutil.Block with the height set. +func dbFetchBlockByHash(dbTx database.Tx, hash *wire.ShaHash) (*btcutil.Block, error) { + // First find the height associated with the provided hash in the index. + blockHeight, err := dbFetchHeightByHash(dbTx, hash) + if err != nil { + return nil, err + } + + // Load the raw block bytes from the database. + blockBytes, err := dbTx.FetchBlock(hash) + if err != nil { + return nil, err + } + + // Create the encapsulated block and set the height appropriately. + block, err := btcutil.NewBlockFromBytes(blockBytes) + if err != nil { + return nil, err + } + block.SetHeight(blockHeight) + + return block, nil +} + +// dbFetchBlockByHeight uses an existing database transaction to retrieve the +// raw block for the provided height, deserialize it, and return a btcutil.Block +// with the height set. +func dbFetchBlockByHeight(dbTx database.Tx, height int32) (*btcutil.Block, error) { + // First find the hash associated with the provided height in the index. + hash, err := dbFetchHashByHeight(dbTx, height) + if err != nil { + return nil, err + } + + // Load the raw block bytes from the database. + blockBytes, err := dbTx.FetchBlock(hash) + if err != nil { + return nil, err + } + + // Create the encapsulated block and set the height appropriately. + block, err := btcutil.NewBlockFromBytes(blockBytes) + if err != nil { + return nil, err + } + block.SetHeight(height) + + return block, nil +} + +// dbMainChainHasBlock uses an existing database transaction to return whether +// or not the main chain contains the block identified by the provided hash. +func dbMainChainHasBlock(dbTx database.Tx, hash *wire.ShaHash) bool { + hashIndex := dbTx.Metadata().Bucket(hashIndexBucketName) + return hashIndex.Get(hash[:]) != nil +} + +// BlockHeightByHash returns the height of the block with the given hash in the +// main chain. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockHeightByHash(hash *wire.ShaHash) (int32, error) { + var height int32 + err := b.db.View(func(dbTx database.Tx) error { + var err error + height, err = dbFetchHeightByHash(dbTx, hash) + return err + }) + return height, err +} + +// BlockHashByHeight returns the hash of the block at the given height in the +// main chain. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockHashByHeight(blockHeight int32) (*wire.ShaHash, error) { + var hash *wire.ShaHash + err := b.db.View(func(dbTx database.Tx) error { + var err error + hash, err = dbFetchHashByHeight(dbTx, blockHeight) + return err + }) + return hash, err +} + +// BlockByHeight returns the block at the given height in the main chain. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockByHeight(blockHeight int32) (*btcutil.Block, error) { + var block *btcutil.Block + err := b.db.View(func(dbTx database.Tx) error { + var err error + block, err = dbFetchBlockByHeight(dbTx, blockHeight) + return err + }) + return block, err +} + +// BlockByHash returns the block from the main chain with the given hash with +// the appropriate chain height set. +// +// This function is safe for concurrent access. +func (b *BlockChain) BlockByHash(hash *wire.ShaHash) (*btcutil.Block, error) { + var block *btcutil.Block + err := b.db.View(func(dbTx database.Tx) error { + var err error + block, err = dbFetchBlockByHash(dbTx, hash) + return err + }) + return block, err +} + +// HeightRange returns a range of block hashes for the given start and end +// heights. It is inclusive of the start height and exclusive of the end +// height. The end height will be limited to the current main chain height. +// +// This function is safe for concurrent access. +func (b *BlockChain) HeightRange(startHeight, endHeight int32) ([]wire.ShaHash, error) { + // Ensure requested heights are sane. + if startHeight < 0 { + return nil, fmt.Errorf("start height of fetch range must not "+ + "be less than zero - got %d", startHeight) + } + if endHeight < startHeight { + return nil, fmt.Errorf("end height of fetch range must not "+ + "be less than the start height - got start %d, end %d", + startHeight, endHeight) + } + + // There is nothing to do when the start and end heights are the same, + // so return now to avoid the chain lock and a database transaction. + if startHeight == endHeight { + return nil, nil + } + + // Grab a lock on the chain to prevent it from changing due to a reorg + // while building the hashes. + b.chainLock.RLock() + defer b.chainLock.RUnlock() + + // When the requested start height is after the most recent best chain + // height, there is nothing to do. + latestHeight := b.bestNode.height + if startHeight > latestHeight { + return nil, nil + } + + // Limit the ending height to the latest height of the chain. + if endHeight > latestHeight+1 { + endHeight = latestHeight + 1 + } + + // Fetch as many as are available within the specified range. + var hashList []wire.ShaHash + err := b.db.View(func(dbTx database.Tx) error { + hashes := make([]wire.ShaHash, 0, endHeight-startHeight) + for i := startHeight; i < endHeight; i++ { + hash, err := dbFetchHashByHeight(dbTx, i) + if err != nil { + return err + } + hashes = append(hashes, *hash) + } + + // Set the list to be returned to the constructed list. + hashList = hashes + return nil + }) + return hashList, err +} diff --git a/blockchain/chainio_test.go b/blockchain/chainio_test.go new file mode 100644 index 00000000..fd9a5ebf --- /dev/null +++ b/blockchain/chainio_test.go @@ -0,0 +1,1030 @@ +// Copyright (c) 2015-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "bytes" + "errors" + "math/big" + "reflect" + "testing" + + database "github.com/btcsuite/btcd/database2" + "github.com/btcsuite/btcd/wire" +) + +// TestErrNotInMainChain ensures the functions related to errNotInMainChain work +// as expected. +func TestErrNotInMainChain(t *testing.T) { + errStr := "no block at height 1 exists" + err := error(errNotInMainChain(errStr)) + + // Ensure the stringized output for the error is as expected. + if err.Error() != errStr { + t.Fatalf("errNotInMainChain retuned unexpected error string - "+ + "got %q, want %q", err.Error(), errStr) + } + + // Ensure error is detected as the correct type. + if !isNotInMainChainErr(err) { + t.Fatalf("isNotInMainChainErr did not detect as expected type") + } + err = errors.New("something else") + if isNotInMainChainErr(err) { + t.Fatalf("isNotInMainChainErr detected incorrect type") + } +} + +// maybeDecompress decompresses the amount and public key script fields of the +// stxo and marks it decompressed if needed. +func (o *spentTxOut) maybeDecompress(version int32) { + // Nothing to do if it's not compressed. + if !o.compressed { + return + } + + o.amount = int64(decompressTxOutAmount(uint64(o.amount))) + o.pkScript = decompressScript(o.pkScript, version) + o.compressed = false +} + +// TestStxoSerialization ensures serializing and deserializing spent transaction +// output entries works as expected. +func TestStxoSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stxo spentTxOut + txVersion int32 // When the txout is not fully spent. + serialized []byte + }{ + // From block 170 in main blockchain. + { + name: "Spends last output of coinbase", + stxo: spentTxOut{ + amount: 5000000000, + pkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"), + isCoinBase: true, + height: 9, + version: 1, + }, + serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"), + }, + // Adapted from block 100025 in main blockchain. + { + name: "Spends last output of non coinbase", + stxo: spentTxOut{ + amount: 13761000000, + pkScript: hexToBytes("76a914b2fb57eadf61e106a100a7445a8c3f67898841ec88ac"), + isCoinBase: false, + height: 100024, + version: 1, + }, + serialized: hexToBytes("8b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec"), + }, + // Adapted from block 100025 in main blockchain. + { + name: "Does not spend last output", + stxo: spentTxOut{ + amount: 34405000000, + pkScript: hexToBytes("76a9146edbc6c4d31bae9f1ccc38538a114bf42de65e8688ac"), + version: 1, + }, + txVersion: 1, + serialized: hexToBytes("0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e86"), + }, + } + + for _, test := range tests { + // Ensure the function to calculate the serialized size without + // actually serializing it is calculated properly. + gotSize := spentTxOutSerializeSize(&test.stxo) + if gotSize != len(test.serialized) { + t.Errorf("spentTxOutSerializeSize (%s): did not get "+ + "expected size - got %d, want %d", test.name, + gotSize, len(test.serialized)) + continue + } + + // Ensure the stxo serializes to the expected value. + gotSerialized := make([]byte, gotSize) + gotBytesWritten := putSpentTxOut(gotSerialized, &test.stxo) + if !bytes.Equal(gotSerialized, test.serialized) { + t.Errorf("putSpentTxOut (%s): did not get expected "+ + "bytes - got %x, want %x", test.name, + gotSerialized, test.serialized) + continue + } + if gotBytesWritten != len(test.serialized) { + t.Errorf("putSpentTxOut (%s): did not get expected "+ + "number of bytes written - got %d, want %d", + test.name, gotBytesWritten, + len(test.serialized)) + continue + } + + // Ensure the serialized bytes are decoded back to the expected + // stxo. + var gotStxo spentTxOut + gotBytesRead, err := decodeSpentTxOut(test.serialized, &gotStxo, + test.txVersion) + if err != nil { + t.Errorf("decodeSpentTxOut (%s): unexpected error: %v", + test.name, err) + continue + } + gotStxo.maybeDecompress(test.stxo.version) + if !reflect.DeepEqual(gotStxo, test.stxo) { + t.Errorf("decodeSpentTxOut (%s) mismatched entries - "+ + "got %v, want %v", test.name, gotStxo, test.stxo) + continue + } + if gotBytesRead != len(test.serialized) { + t.Errorf("decodeSpentTxOut (%s): did not get expected "+ + "number of bytes read - got %d, want %d", + test.name, gotBytesRead, len(test.serialized)) + continue + } + } +} + +// TestStxoDecodeErrors performs negative tests against decoding spent +// transaction outputs to ensure error paths work as expected. +func TestStxoDecodeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stxo spentTxOut + txVersion int32 // When the txout is not fully spent. + serialized []byte + bytesRead int // Expected number of bytes read. + errType error + }{ + { + name: "nothing serialized", + stxo: spentTxOut{}, + serialized: hexToBytes(""), + errType: errDeserialize(""), + bytesRead: 0, + }, + { + name: "no data after header code w/o version", + stxo: spentTxOut{}, + serialized: hexToBytes("00"), + errType: errDeserialize(""), + bytesRead: 1, + }, + { + name: "no data after header code with version", + stxo: spentTxOut{}, + serialized: hexToBytes("13"), + errType: errDeserialize(""), + bytesRead: 1, + }, + { + name: "no data after version", + stxo: spentTxOut{}, + serialized: hexToBytes("1301"), + errType: errDeserialize(""), + bytesRead: 2, + }, + { + name: "no serialized tx version and passed 0", + stxo: spentTxOut{}, + serialized: hexToBytes("003205"), + errType: AssertError(""), + bytesRead: 1, + }, + { + name: "incomplete compressed txout", + stxo: spentTxOut{}, + txVersion: 1, + serialized: hexToBytes("0032"), + errType: errDeserialize(""), + bytesRead: 2, + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned. + gotBytesRead, err := decodeSpentTxOut(test.serialized, + &test.stxo, test.txVersion) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("decodeSpentTxOut (%s): expected error type "+ + "does not match - got %T, want %T", test.name, + err, test.errType) + continue + } + + // Ensure the expected number of bytes read is returned. + if gotBytesRead != test.bytesRead { + t.Errorf("decodeSpentTxOut (%s): unexpected number of "+ + "bytes read - got %d, want %d", test.name, + gotBytesRead, test.bytesRead) + continue + } + } +} + +// TestSpendJournalSerialization ensures serializing and deserializing spend +// journal entries works as expected. +func TestSpendJournalSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + entry []spentTxOut + blockTxns []*wire.MsgTx + utxoView *UtxoViewpoint + serialized []byte + }{ + // From block 2 in main blockchain. + { + name: "No spends", + entry: nil, + blockTxns: nil, + utxoView: NewUtxoViewpoint(), + serialized: nil, + }, + // From block 170 in main blockchain. + { + name: "One tx with one input spends last output of coinbase", + entry: []spentTxOut{{ + amount: 5000000000, + pkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"), + isCoinBase: true, + height: 9, + version: 1, + }}, + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9"), + Index: 0, + }, + SignatureScript: hexToBytes("47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901"), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 1000000000, + PkScript: hexToBytes("4104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84cac"), + }, { + Value: 4000000000, + PkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"), + }}, + LockTime: 0, + }}, + utxoView: NewUtxoViewpoint(), + serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"), + }, + // Adapted from block 100025 in main blockchain. + { + name: "Two txns when one spends last output, one doesn't", + entry: []spentTxOut{{ + amount: 34405000000, + pkScript: hexToBytes("76a9146edbc6c4d31bae9f1ccc38538a114bf42de65e8688ac"), + version: 1, + }, { + amount: 13761000000, + pkScript: hexToBytes("76a914b2fb57eadf61e106a100a7445a8c3f67898841ec88ac"), + isCoinBase: false, + height: 100024, + version: 1, + }}, + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"), + Index: 1, + }, + SignatureScript: hexToBytes("493046022100c167eead9840da4a033c9a56470d7794a9bb1605b377ebe5688499b39f94be59022100fb6345cab4324f9ea0b9ee9169337534834638d818129778370f7d378ee4a325014104d962cac5390f12ddb7539507065d0def320d68c040f2e73337c3a1aaaab7195cb5c4d02e0959624d534f3c10c3cf3d73ca5065ebd62ae986b04c6d090d32627c"), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 5000000, + PkScript: hexToBytes("76a914f419b8db4ba65f3b6fcc233acb762ca6f51c23d488ac"), + }, { + Value: 34400000000, + PkScript: hexToBytes("76a914cadf4fc336ab3c6a4610b75f31ba0676b7f663d288ac"), + }}, + LockTime: 0, + }, { + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("92fbe1d4be82f765dfabc9559d4620864b05cc897c4db0e29adac92d294e52b7"), + Index: 0, + }, + SignatureScript: hexToBytes("483045022100e256743154c097465cf13e89955e1c9ff2e55c46051b627751dee0144183157e02201d8d4f02cde8496aae66768f94d35ce54465bd4ae8836004992d3216a93a13f00141049d23ce8686fe9b802a7a938e8952174d35dd2c2089d4112001ed8089023ab4f93a3c9fcd5bfeaa9727858bf640dc1b1c05ec3b434bb59837f8640e8810e87742"), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 5000000, + PkScript: hexToBytes("76a914a983ad7c92c38fc0e2025212e9f972204c6e687088ac"), + }, { + Value: 13756000000, + PkScript: hexToBytes("76a914a6ebd69952ab486a7a300bfffdcb395dc7d47c2388ac"), + }}, + LockTime: 0, + }}, + utxoView: &UtxoViewpoint{entries: map[wire.ShaHash]*UtxoEntry{ + *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"): { + version: 1, + isCoinBase: false, + blockHeight: 100024, + sparseOutputs: map[uint32]*utxoOutput{ + 1: { + amount: 34405000000, + pkScript: hexToBytes("76a9142084541c3931677527a7eafe56fd90207c344eb088ac"), + }, + }, + }, + }}, + serialized: hexToBytes("8b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e86"), + }, + // Hand crafted. + { + name: "One tx, two inputs from same tx, neither spend last output", + entry: []spentTxOut{{ + amount: 165125632, + pkScript: hexToBytes("51"), + version: 1, + }, { + amount: 154370000, + pkScript: hexToBytes("51"), + version: 1, + }}, + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"), + Index: 1, + }, + SignatureScript: hexToBytes(""), + Sequence: 0xffffffff, + }, { + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"), + Index: 2, + }, + SignatureScript: hexToBytes(""), + Sequence: 0xffffffff, + }}, + TxOut: []*wire.TxOut{{ + Value: 165125632, + PkScript: hexToBytes("51"), + }, { + Value: 154370000, + PkScript: hexToBytes("51"), + }}, + LockTime: 0, + }}, + utxoView: &UtxoViewpoint{entries: map[wire.ShaHash]*UtxoEntry{ + *newShaHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"): { + version: 1, + isCoinBase: false, + blockHeight: 100000, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 165712179, + pkScript: hexToBytes("51"), + }, + }, + }, + }}, + serialized: hexToBytes("0087bc3707510084c3d19a790751"), + }, + } + + for i, test := range tests { + // Ensure the journal entry serializes to the expected value. + gotBytes := serializeSpendJournalEntry(test.entry) + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeSpendJournalEntry #%d (%s): "+ + "mismatched bytes - got %x, want %x", i, + test.name, gotBytes, test.serialized) + continue + } + + // Deserialize to a spend journal entry. + gotEntry, err := deserializeSpendJournalEntry(test.serialized, + test.blockTxns, test.utxoView) + if err != nil { + t.Errorf("deserializeSpendJournalEntry #%d (%s) "+ + "unexpected error: %v", i, test.name, err) + continue + } + for stxoIdx := range gotEntry { + stxo := &gotEntry[stxoIdx] + stxo.maybeDecompress(test.entry[stxoIdx].version) + } + + // Ensure that the deserialized spend journal entry has the + // correct properties. + if !reflect.DeepEqual(gotEntry, test.entry) { + t.Errorf("deserializeSpendJournalEntry #%d (%s) "+ + "mismatched entries - got %v, want %v", + i, test.name, gotEntry, test.entry) + continue + } + } +} + +// TestSpendJournalErrors performs negative tests against deserializing spend +// journal entries to ensure error paths work as expected. +func TestSpendJournalErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + blockTxns []*wire.MsgTx + utxoView *UtxoViewpoint + serialized []byte + errType error + }{ + // Adapted from block 170 in main blockchain. + { + name: "Force assertion due to missing stxos", + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9"), + Index: 0, + }, + SignatureScript: hexToBytes("47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901"), + Sequence: 0xffffffff, + }}, + LockTime: 0, + }}, + utxoView: NewUtxoViewpoint(), + serialized: hexToBytes(""), + errType: AssertError(""), + }, + { + name: "Force deserialization error in stxos", + blockTxns: []*wire.MsgTx{{ // Coinbase omitted. + Version: 1, + TxIn: []*wire.TxIn{{ + PreviousOutPoint: wire.OutPoint{ + Hash: *newShaHashFromStr("0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9"), + Index: 0, + }, + SignatureScript: hexToBytes("47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901"), + Sequence: 0xffffffff, + }}, + LockTime: 0, + }}, + utxoView: NewUtxoViewpoint(), + serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a"), + errType: errDeserialize(""), + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned and the returned + // slice is nil. + stxos, err := deserializeSpendJournalEntry(test.serialized, + test.blockTxns, test.utxoView) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("deserializeSpendJournalEntry (%s): expected "+ + "error type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if stxos != nil { + t.Errorf("deserializeSpendJournalEntry (%s): returned "+ + "slice of spent transaction outputs is not nil", + test.name) + continue + } + } +} + +// TestUtxoSerialization ensures serializing and deserializing unspent +// trasaction output entries works as expected. +func TestUtxoSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + entry *UtxoEntry + serialized []byte + }{ + // From tx in main blockchain: + // 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098 + { + name: "Only output 0, coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: true, + blockHeight: 1, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 5000000000, + pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"), + }, + }, + }, + serialized: hexToBytes("010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"), + }, + // From tx in main blockchain: + // 8131ffb0a2c945ecaf9b9063e59558784f9c3a74741ce6ae2a18d0571dac15bb + { + name: "Only output 1, not coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 100001, + sparseOutputs: map[uint32]*utxoOutput{ + 1: { + amount: 1000000, + pkScript: hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"), + }, + }, + }, + serialized: hexToBytes("01858c21040700ee8bd501094a7d5ca318da2506de35e1cb025ddc"), + }, + // Adapted from tx in main blockchain: + // df3f3f442d9699857f7f49de4ff0b5d0f3448bec31cdc7b5bf6d25f2abd637d5 + { + name: "Only output 2, coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: true, + blockHeight: 99004, + sparseOutputs: map[uint32]*utxoOutput{ + 2: { + amount: 100937281, + pkScript: hexToBytes("76a914da33f77cee27c2a975ed5124d7e4f7f97513510188ac"), + }, + }, + }, + serialized: hexToBytes("0185843c010182b095bf4100da33f77cee27c2a975ed5124d7e4f7f975135101"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2 not coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 2: { + amount: 15000000, + pkScript: hexToBytes("76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2, not coinbase, 1 marked spent", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 1: { // This won't be serialized. + spent: true, + amount: 1000000, + pkScript: hexToBytes("76a914e43031c3e46f20bf1ccee9553ce815de5a48467588ac"), + }, + 2: { + amount: 15000000, + pkScript: hexToBytes("76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2, not coinbase, output 2 compressed", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 2: { + // Uncompressed Amount: 15000000 + // Uncompressed PkScript: 76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac + compressed: true, + amount: 137, + pkScript: hexToBytes("00b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // Adapted from tx in main blockchain: + // 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f + { + name: "outputs 0 and 2, not coinbase, output 2 compressed, packed indexes reversed", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 113931, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + amount: 20000000, + pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"), + }, + 2: { + // Uncompressed Amount: 15000000 + // Uncompressed PkScript: 76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac + compressed: true, + amount: 137, + pkScript: hexToBytes("00b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + }, + }, + serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"), + }, + // From tx in main blockchain: + // 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098 + { + name: "Only output 0, coinbase, fully spent", + entry: &UtxoEntry{ + version: 1, + isCoinBase: true, + blockHeight: 1, + sparseOutputs: map[uint32]*utxoOutput{ + 0: { + spent: true, + amount: 5000000000, + pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"), + }, + }, + }, + serialized: nil, + }, + // Adapted from tx in main blockchain: + // 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620 + { + name: "Only output 22, not coinbase", + entry: &UtxoEntry{ + version: 1, + isCoinBase: false, + blockHeight: 338156, + sparseOutputs: map[uint32]*utxoOutput{ + 22: { + spent: false, + amount: 366875659, + pkScript: hexToBytes("a9141dd46a006572d820e448e12d2bbb38640bc718e687"), + }, + }, + }, + serialized: hexToBytes("0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6"), + }, + } + + for i, test := range tests { + // Ensure the utxo entry serializes to the expected value. + gotBytes, err := serializeUtxoEntry(test.entry) + if err != nil { + t.Errorf("serializeUtxoEntry #%d (%s) unexpected "+ + "error: %v", i, test.name, err) + continue + } + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeUtxoEntry #%d (%s): mismatched "+ + "bytes - got %x, want %x", i, test.name, + gotBytes, test.serialized) + continue + } + + // Don't try to deserialize if the test entry was fully spent + // since it will have a nil serialization. + if test.entry.IsFullySpent() { + continue + } + + // Deserialize to a utxo entry. + utxoEntry, err := deserializeUtxoEntry(test.serialized) + if err != nil { + t.Errorf("deserializeUtxoEntry #%d (%s) unexpected "+ + "error: %v", i, test.name, err) + continue + } + + // Ensure that the deserialized utxo entry has the same + // properties for the containing transaction and block height. + if utxoEntry.Version() != test.entry.Version() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "version: got %d, want %d", i, test.name, + utxoEntry.Version(), test.entry.Version()) + continue + } + if utxoEntry.IsCoinBase() != test.entry.IsCoinBase() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "coinbase flag: got %v, want %v", i, test.name, + utxoEntry.IsCoinBase(), test.entry.IsCoinBase()) + continue + } + if utxoEntry.BlockHeight() != test.entry.BlockHeight() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "block height: got %d, want %d", i, test.name, + utxoEntry.BlockHeight(), + test.entry.BlockHeight()) + continue + } + if utxoEntry.IsFullySpent() != test.entry.IsFullySpent() { + t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+ + "fully spent: got %v, want %v", i, test.name, + utxoEntry.IsFullySpent(), + test.entry.IsFullySpent()) + continue + } + + // Ensure all of the outputs in the test entry match the + // spentness of the output in the deserialized entry and the + // deserialized entry does not contain any additional utxos. + var numUnspent int + for outputIndex := range test.entry.sparseOutputs { + gotSpent := utxoEntry.IsOutputSpent(outputIndex) + wantSpent := test.entry.IsOutputSpent(outputIndex) + if !wantSpent { + numUnspent++ + } + if gotSpent != wantSpent { + t.Errorf("deserializeUtxoEntry #%d (%s) output "+ + "#%d: mismatched spent: got %v, want "+ + "%v", i, test.name, outputIndex, + gotSpent, wantSpent) + continue + + } + } + if len(utxoEntry.sparseOutputs) != numUnspent { + t.Errorf("deserializeUtxoEntry #%d (%s): mismatched "+ + "number of unspent outputs: got %d, want %d", i, + test.name, len(utxoEntry.sparseOutputs), + numUnspent) + continue + } + + // Ensure all of the amounts and scripts of the utxos in the + // deserialized entry match the ones in the test entry. + for outputIndex := range utxoEntry.sparseOutputs { + gotAmount := utxoEntry.AmountByIndex(outputIndex) + wantAmount := test.entry.AmountByIndex(outputIndex) + if gotAmount != wantAmount { + t.Errorf("deserializeUtxoEntry #%d (%s) "+ + "output #%d: mismatched amounts: got "+ + "%d, want %d", i, test.name, + outputIndex, gotAmount, wantAmount) + continue + } + + gotPkScript := utxoEntry.PkScriptByIndex(outputIndex) + wantPkScript := test.entry.PkScriptByIndex(outputIndex) + if !bytes.Equal(gotPkScript, wantPkScript) { + t.Errorf("deserializeUtxoEntry #%d (%s) "+ + "output #%d mismatched scripts: got "+ + "%x, want %x", i, test.name, + outputIndex, gotPkScript, wantPkScript) + continue + } + } + } +} + +// TestUtxoEntryHeaderCodeErrors performs negative tests against unspent +// transaction output header codes to ensure error paths work as expected. +func TestUtxoEntryHeaderCodeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + entry *UtxoEntry + code uint64 + bytesRead int // Expected number of bytes read. + errType error + }{ + { + name: "Force assertion due to fully spent tx", + entry: &UtxoEntry{}, + errType: AssertError(""), + bytesRead: 0, + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned and the code is 0. + code, gotBytesRead, err := utxoEntryHeaderCode(test.entry, 0) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("utxoEntryHeaderCode (%s): expected error "+ + "type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if code != 0 { + t.Errorf("utxoEntryHeaderCode (%s): unexpected code "+ + "on error - got %d, want 0", test.name, code) + continue + } + + // Ensure the expected number of bytes read is returned. + if gotBytesRead != test.bytesRead { + t.Errorf("utxoEntryHeaderCode (%s): unexpected number "+ + "of bytes read - got %d, want %d", test.name, + gotBytesRead, test.bytesRead) + continue + } + } +} + +// TestUtxoEntryDeserializeErrors performs negative tests against deserializing +// unspent transaction outputs to ensure error paths work as expected. +func TestUtxoEntryDeserializeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serialized []byte + errType error + }{ + { + name: "no data after version", + serialized: hexToBytes("01"), + errType: errDeserialize(""), + }, + { + name: "no data after block height", + serialized: hexToBytes("0101"), + errType: errDeserialize(""), + }, + { + name: "no data after header code", + serialized: hexToBytes("010102"), + errType: errDeserialize(""), + }, + { + name: "not enough bytes for unspentness bitmap", + serialized: hexToBytes("01017800"), + errType: errDeserialize(""), + }, + { + name: "incomplete compressed txout", + serialized: hexToBytes("01010232"), + errType: errDeserialize(""), + }, + } + + for _, test := range tests { + // Ensure the expected error type is returned and the returned + // entry is nil. + entry, err := deserializeUtxoEntry(test.serialized) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("deserializeUtxoEntry (%s): expected error "+ + "type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if entry != nil { + t.Errorf("deserializeUtxoEntry (%s): returned entry "+ + "is not nil", test.name) + continue + } + } +} + +// TestBestChainStateSerialization ensures serializing and deserializing the +// best chain state works as expected. +func TestBestChainStateSerialization(t *testing.T) { + t.Parallel() + + workSum := new(big.Int) + tests := []struct { + name string + state bestChainState + serialized []byte + }{ + { + name: "genesis", + state: bestChainState{ + hash: *newShaHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"), + height: 0, + totalTxns: 1, + workSum: func() *big.Int { + workSum.Add(workSum, CalcWork(486604799)) + return new(big.Int).Set(workSum) + }(), // 0x0100010001 + }, + serialized: hexToBytes("6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000000000000100000000000000050000000100010001"), + }, + { + name: "block 1", + state: bestChainState{ + hash: *newShaHashFromStr("00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048"), + height: 1, + totalTxns: 2, + workSum: func() *big.Int { + workSum.Add(workSum, CalcWork(486604799)) + return new(big.Int).Set(workSum) + }(), // 0x0200020002 + }, + serialized: hexToBytes("4860eb18bf1b1620e37e9490fc8a427514416fd75159ab86688e9a8300000000010000000200000000000000050000000200020002"), + }, + } + + for i, test := range tests { + // Ensure the state serializes to the expected value. + gotBytes := serializeBestChainState(test.state) + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeBestChainState #%d (%s): mismatched "+ + "bytes - got %x, want %x", i, test.name, + gotBytes, test.serialized) + continue + } + + // Ensure the serialized bytes are decoded back to the expected + // state. + state, err := deserializeBestChainState(test.serialized) + if err != nil { + t.Errorf("deserializeBestChainState #%d (%s) "+ + "unexpected error: %v", i, test.name, err) + continue + } + if !reflect.DeepEqual(state, test.state) { + t.Errorf("deserializeBestChainState #%d (%s) "+ + "mismatched state - got %v, want %v", i, + test.name, state, test.state) + continue + + } + } +} + +// TestBestChainStateDeserializeErrors performs negative tests against +// deserializing the chain state to ensure error paths work as expected. +func TestBestChainStateDeserializeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serialized []byte + errType error + }{ + { + name: "nothing serialized", + serialized: hexToBytes(""), + errType: database.Error{ErrorCode: database.ErrCorruption}, + }, + { + name: "short data in hash", + serialized: hexToBytes("0000"), + errType: database.Error{ErrorCode: database.ErrCorruption}, + }, + { + name: "short data in work sum", + serialized: hexToBytes("6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000001000000000000000500000001000100"), + errType: database.Error{ErrorCode: database.ErrCorruption}, + }, + } + + for _, test := range tests { + // Ensure the expected error type and code is returned. + _, err := deserializeBestChainState(test.serialized) + if reflect.TypeOf(err) != reflect.TypeOf(test.errType) { + t.Errorf("deserializeBestChainState (%s): expected "+ + "error type does not match - got %T, want %T", + test.name, err, test.errType) + continue + } + if derr, ok := err.(database.Error); ok { + tderr := test.errType.(database.Error) + if derr.ErrorCode != tderr.ErrorCode { + t.Errorf("deserializeBestChainState (%s): "+ + "wrong error code got: %v, want: %v", + test.name, derr.ErrorCode, + tderr.ErrorCode) + continue + } + } + } +} diff --git a/blockchain/checkpoints.go b/blockchain/checkpoints.go index c0dc21c2..b3e49943 100644 --- a/blockchain/checkpoints.go +++ b/blockchain/checkpoints.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -8,6 +8,7 @@ import ( "fmt" "github.com/btcsuite/btcd/chaincfg" + database "github.com/btcsuite/btcd/database2" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" @@ -29,14 +30,23 @@ func newShaHashFromStr(hexStr string) *wire.ShaHash { // DisableCheckpoints provides a mechanism to disable validation against // checkpoints which you DO NOT want to do in production. It is provided only // for debug purposes. +// +// This function is safe for concurrent access. func (b *BlockChain) DisableCheckpoints(disable bool) { + b.chainLock.Lock() b.noCheckpoints = disable + b.chainLock.Unlock() } // Checkpoints returns a slice of checkpoints (regardless of whether they are // already known). When checkpoints are disabled or there are no checkpoints // for the active network, it will return nil. +// +// This function is safe for concurrent access. func (b *BlockChain) Checkpoints() []chaincfg.Checkpoint { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return nil } @@ -44,10 +54,12 @@ func (b *BlockChain) Checkpoints() []chaincfg.Checkpoint { return b.chainParams.Checkpoints } -// LatestCheckpoint returns the most recent checkpoint (regardless of whether it +// latestCheckpoint returns the most recent checkpoint (regardless of whether it // is already known). When checkpoints are disabled or there are no checkpoints // for the active network, it will return nil. -func (b *BlockChain) LatestCheckpoint() *chaincfg.Checkpoint { +// +// This function MUST be called with the chain state lock held (for reads). +func (b *BlockChain) latestCheckpoint() *chaincfg.Checkpoint { if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return nil } @@ -56,9 +68,23 @@ func (b *BlockChain) LatestCheckpoint() *chaincfg.Checkpoint { return &checkpoints[len(checkpoints)-1] } +// LatestCheckpoint returns the most recent checkpoint (regardless of whether it +// is already known). When checkpoints are disabled or there are no checkpoints +// for the active network, it will return nil. +// +// This function is safe for concurrent access. +func (b *BlockChain) LatestCheckpoint() *chaincfg.Checkpoint { + b.chainLock.RLock() + checkpoint := b.latestCheckpoint() + b.chainLock.RUnlock() + return checkpoint +} + // verifyCheckpoint returns whether the passed block height and hash combination // match the hard-coded checkpoint data. It also returns true if there is no // checkpoint data for the passed block height. +// +// This function MUST be called with the chain lock held (for reads). func (b *BlockChain) verifyCheckpoint(height int32, hash *wire.ShaHash) bool { if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return true @@ -83,6 +109,8 @@ func (b *BlockChain) verifyCheckpoint(height int32, hash *wire.ShaHash) bool { // available in the downloaded portion of the block chain and returns the // associated block. It returns nil if a checkpoint can't be found (this should // really only happen for blocks before the first checkpoint). +// +// This function MUST be called with the chain lock held (for reads). func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { if b.noCheckpoints || len(b.chainParams.Checkpoints) == 0 { return nil, nil @@ -98,20 +126,21 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // Perform the initial search to find and cache the latest known // checkpoint if the best chain is not known yet or we haven't already // previously searched. - if b.bestChain == nil || (b.checkpointBlock == nil && b.nextCheckpoint == nil) { + if b.checkpointBlock == nil && b.nextCheckpoint == nil { // Loop backwards through the available checkpoints to find one - // that we already have. + // that is already available. checkpointIndex := -1 - for i := numCheckpoints - 1; i >= 0; i-- { - exists, err := b.db.ExistsSha(checkpoints[i].Hash) - if err != nil { - return nil, err - } - - if exists { - checkpointIndex = i - break + err := b.db.View(func(dbTx database.Tx) error { + for i := numCheckpoints - 1; i >= 0; i-- { + if dbMainChainHasBlock(dbTx, checkpoints[i].Hash) { + checkpointIndex = i + break + } } + return nil + }) + if err != nil { + return nil, err } // No known latest checkpoint. This will only happen on blocks @@ -125,19 +154,26 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // Cache the latest known checkpoint block for future lookups. checkpoint := checkpoints[checkpointIndex] - block, err := b.db.FetchBlockBySha(checkpoint.Hash) + err = b.db.View(func(dbTx database.Tx) error { + block, err := dbFetchBlockByHash(dbTx, checkpoint.Hash) + if err != nil { + return err + } + b.checkpointBlock = block + + // Set the next expected checkpoint block accordingly. + b.nextCheckpoint = nil + if checkpointIndex < numCheckpoints-1 { + b.nextCheckpoint = &checkpoints[checkpointIndex+1] + } + + return nil + }) if err != nil { return nil, err } - b.checkpointBlock = block - // Set the next expected checkpoint block accordingly. - b.nextCheckpoint = nil - if checkpointIndex < numCheckpoints-1 { - b.nextCheckpoint = &checkpoints[checkpointIndex+1] - } - - return block, nil + return b.checkpointBlock, nil } // At this point we've already searched for the latest known checkpoint, @@ -150,7 +186,7 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // When there is a next checkpoint and the height of the current best // chain does not exceed it, the current checkpoint lockin is still // the latest known checkpoint. - if b.bestChain.height < b.nextCheckpoint.Height { + if b.bestNode.height < b.nextCheckpoint.Height { return b.checkpointBlock, nil } @@ -163,11 +199,17 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // that if this lookup fails something is very wrong since the chain // has already passed the checkpoint which was verified as accurate // before inserting it. - block, err := b.db.FetchBlockBySha(b.nextCheckpoint.Hash) + err := b.db.View(func(tx database.Tx) error { + block, err := dbFetchBlockByHash(tx, b.nextCheckpoint.Hash) + if err != nil { + return err + } + b.checkpointBlock = block + return nil + }) if err != nil { return nil, err } - b.checkpointBlock = block // Set the next expected checkpoint. checkpointIndex := -1 @@ -188,8 +230,6 @@ func (b *BlockChain) findPreviousCheckpoint() (*btcutil.Block, error) { // isNonstandardTransaction determines whether a transaction contains any // scripts which are not one of the standard types. func isNonstandardTransaction(tx *btcutil.Tx) bool { - // TODO(davec): Should there be checks for the input signature scripts? - // Check all of the output public key scripts for non-standard scripts. for _, txOut := range tx.MsgTx().TxOut { scriptClass := txscript.GetScriptClass(txOut.PkScript) @@ -215,66 +255,80 @@ func isNonstandardTransaction(tx *btcutil.Tx) bool { // // The intent is that candidates are reviewed by a developer to make the final // decision and then manually added to the list of checkpoints for a network. +// +// This function is safe for concurrent access. func (b *BlockChain) IsCheckpointCandidate(block *btcutil.Block) (bool, error) { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + // Checkpoints must be enabled. if b.noCheckpoints { return false, fmt.Errorf("checkpoints are disabled") } - // A checkpoint must be in the main chain. - exists, err := b.db.ExistsSha(block.Sha()) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - - // A checkpoint must be at least CheckpointConfirmations blocks before - // the end of the main chain. - blockHeight := block.Height() - _, mainChainHeight, err := b.db.NewestSha() - if err != nil { - return false, err - } - if blockHeight > (mainChainHeight - CheckpointConfirmations) { - return false, nil - } - - // Get the previous block. - prevHash := &block.MsgBlock().Header.PrevBlock - prevBlock, err := b.db.FetchBlockBySha(prevHash) - if err != nil { - return false, err - } - - // Get the next block. - nextHash, err := b.db.FetchBlockShaByHeight(blockHeight + 1) - if err != nil { - return false, err - } - nextBlock, err := b.db.FetchBlockBySha(nextHash) - if err != nil { - return false, err - } - - // A checkpoint must have timestamps for the block and the blocks on - // either side of it in order (due to the median time allowance this is - // not always the case). - prevTime := prevBlock.MsgBlock().Header.Timestamp - curTime := block.MsgBlock().Header.Timestamp - nextTime := nextBlock.MsgBlock().Header.Timestamp - if prevTime.After(curTime) || nextTime.Before(curTime) { - return false, nil - } - - // A checkpoint must have transactions that only contain standard - // scripts. - for _, tx := range block.Transactions() { - if isNonstandardTransaction(tx) { - return false, nil + var isCandidate bool + err := b.db.View(func(dbTx database.Tx) error { + // A checkpoint must be in the main chain. + blockHeight, err := dbFetchHeightByHash(dbTx, block.Sha()) + if err != nil { + // Only return an error if it's not due to the block not + // being in the main chain. + if !isNotInMainChainErr(err) { + return err + } + return nil } - } - return true, nil + // Ensure the height of the passed block and the entry for the + // block in the main chain match. This should always be the + // case unless the caller provided an invalid block. + if blockHeight != block.Height() { + return fmt.Errorf("passed block height of %d does not "+ + "match the main chain height of %d", + block.Height(), blockHeight) + } + + // A checkpoint must be at least CheckpointConfirmations blocks + // before the end of the main chain. + mainChainHeight := b.bestNode.height + if blockHeight > (mainChainHeight - CheckpointConfirmations) { + return nil + } + + // Get the previous block header. + prevHash := &block.MsgBlock().Header.PrevBlock + prevHeader, err := dbFetchHeaderByHash(dbTx, prevHash) + if err != nil { + return err + } + + // Get the next block header. + nextHeader, err := dbFetchHeaderByHeight(dbTx, blockHeight+1) + if err != nil { + return err + } + + // A checkpoint must have timestamps for the block and the + // blocks on either side of it in order (due to the median time + // allowance this is not always the case). + prevTime := prevHeader.Timestamp + curTime := block.MsgBlock().Header.Timestamp + nextTime := nextHeader.Timestamp + if prevTime.After(curTime) || nextTime.Before(curTime) { + return nil + } + + // A checkpoint must have transactions that only contain + // standard scripts. + for _, tx := range block.Transactions() { + if isNonstandardTransaction(tx) { + return nil + } + } + + // All of the checks passed, so the block is a candidate. + isCandidate = true + return nil + }) + return isCandidate, err } diff --git a/blockchain/common_test.go b/blockchain/common_test.go index 2d8e8c97..b833f820 100644 --- a/blockchain/common_test.go +++ b/blockchain/common_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2013-2016 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,18 +15,21 @@ import ( "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/database" - _ "github.com/btcsuite/btcd/database/ldb" - _ "github.com/btcsuite/btcd/database/memdb" + database "github.com/btcsuite/btcd/database2" + _ "github.com/btcsuite/btcd/database2/ffldb" "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcutil" ) -// testDbType is the database backend type to use for the tests. -const testDbType = "memdb" +const ( + // testDbType is the database backend type to use for the tests. + testDbType = "ffldb" -// testDbRoot is the root directory used to create all test databases. -const testDbRoot = "testdbs" + // testDbRoot is the root directory used to create all test databases. + testDbRoot = "testdbs" + + // blockDataNet is the expected network in the test block data. + blockDataNet = wire.MainNet +) // filesExists returns whether or not the named file or directory exists. func fileExists(name string) bool { @@ -41,9 +44,9 @@ func fileExists(name string) bool { // isSupportedDbType returns whether or not the passed database type is // currently supported. func isSupportedDbType(dbType string) bool { - supportedDBs := database.SupportedDBs() - for _, sDbType := range supportedDBs { - if dbType == sDbType { + supportedDrivers := database.SupportedDrivers() + for _, driver := range supportedDrivers { + if dbType == driver { return true } } @@ -61,10 +64,10 @@ func chainSetup(dbName string) (*blockchain.BlockChain, func(), error) { // Handle memory database specially since it doesn't need the disk // specific handling. - var db database.Db + var db database.DB var teardown func() if testDbType == "memdb" { - ndb, err := database.CreateDB(testDbType) + ndb, err := database.Create(testDbType) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } @@ -88,7 +91,7 @@ func chainSetup(dbName string) (*blockchain.BlockChain, func(), error) { // Create a new database to store the accepted blocks into. dbPath := filepath.Join(testDbRoot, dbName) _ = os.RemoveAll(dbPath) - ndb, err := database.CreateDB(testDbType, dbPath) + ndb, err := database.Create(testDbType, dbPath, blockDataNet) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } @@ -97,39 +100,34 @@ func chainSetup(dbName string) (*blockchain.BlockChain, func(), error) { // Setup a teardown function for cleaning up. This function is // returned to the caller to be invoked when it is done testing. teardown = func() { - dbVersionPath := filepath.Join(testDbRoot, dbName+".ver") - db.Sync() db.Close() os.RemoveAll(dbPath) - os.Remove(dbVersionPath) os.RemoveAll(testDbRoot) } } - // Insert the main network genesis block. This is part of the initial - // database setup. - genesisBlock := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock) - _, err := db.InsertBlock(genesisBlock) + // Create the main chain instance. + chain, err := blockchain.New(&blockchain.Config{ + DB: db, + ChainParams: &chaincfg.MainNetParams, + }) if err != nil { teardown() - err := fmt.Errorf("failed to insert genesis block: %v", err) + err := fmt.Errorf("failed to create chain instance: %v", err) return nil, nil, err } - - chain := blockchain.New(db, &chaincfg.MainNetParams, nil, nil) return chain, teardown, nil } -// loadTxStore returns a transaction store loaded from a file. -func loadTxStore(filename string) (blockchain.TxStore, error) { - // The txstore file format is: - // - // +// loadUtxoView returns a utxo view loaded from a file. +func loadUtxoView(filename string) (*blockchain.UtxoViewpoint, error) { + // The utxostore file format is: + // // - // All num and length fields are little-endian uint32s. The spent bits - // field is padded to a byte boundary. + // The serialized utxo len is a little endian uint32 and the serialized + // utxo uses the format described in chainio.go. - filename = filepath.Join("testdata/", filename) + filename = filepath.Join("testdata", filename) fi, err := os.Open(filename) if err != nil { return nil, err @@ -144,80 +142,40 @@ func loadTxStore(filename string) (blockchain.TxStore, error) { } defer fi.Close() - // Num of transaction store objects. - var numItems uint32 - if err := binary.Read(r, binary.LittleEndian, &numItems); err != nil { - return nil, err - } - - txStore := make(blockchain.TxStore) - var uintBuf uint32 - for height := uint32(0); height < numItems; height++ { - txD := blockchain.TxData{} - - // Serialized transaction length. - err = binary.Read(r, binary.LittleEndian, &uintBuf) + view := blockchain.NewUtxoViewpoint() + for { + // Hash of the utxo entry. + var hash wire.ShaHash + _, err := io.ReadAtLeast(r, hash[:], len(hash[:])) if err != nil { - return nil, err - } - serializedTxLen := uintBuf - if serializedTxLen > wire.MaxBlockPayload { - return nil, fmt.Errorf("Read serialized transaction "+ - "length of %d is larger max allowed %d", - serializedTxLen, wire.MaxBlockPayload) - } - - // Transaction. - var msgTx wire.MsgTx - err = msgTx.Deserialize(r) - if err != nil { - return nil, err - } - txD.Tx = btcutil.NewTx(&msgTx) - - // Transaction hash. - txHash := msgTx.TxSha() - txD.Hash = &txHash - - // Block height the transaction came from. - err = binary.Read(r, binary.LittleEndian, &uintBuf) - if err != nil { - return nil, err - } - txD.BlockHeight = int32(uintBuf) - - // Num spent bits. - err = binary.Read(r, binary.LittleEndian, &uintBuf) - if err != nil { - return nil, err - } - numSpentBits := uintBuf - numSpentBytes := numSpentBits / 8 - if numSpentBits%8 != 0 { - numSpentBytes++ - } - - // Packed spent bytes. - spentBytes := make([]byte, numSpentBytes) - _, err = io.ReadFull(r, spentBytes) - if err != nil { - return nil, err - } - - // Populate spent data based on spent bits. - txD.Spent = make([]bool, numSpentBits) - for byteNum, spentByte := range spentBytes { - for bit := 0; bit < 8; bit++ { - if uint32((byteNum*8)+bit) < numSpentBits { - if spentByte&(1< [0x00] +// 127 -> [0x7f] * Max 1-byte value +// 128 -> [0x80 0x00] +// 129 -> [0x80 0x01] +// 255 -> [0x80 0x7f] +// 256 -> [0x81 0x00] +// 16511 -> [0xff 0x7f] * Max 2-byte value +// 16512 -> [0x80 0x80 0x00] +// 32895 -> [0x80 0xff 0x7f] +// 2113663 -> [0xff 0xff 0x7f] * Max 3-byte value +// 270549119 -> [0xff 0xff 0xff 0x7f] * Max 4-byte value +// 2^64-1 -> [0x80 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0x7f] +// +// References: +// https://en.wikipedia.org/wiki/Variable-length_quantity +// http://www.codecodex.com/wiki/Variable-Length_Integers +// ----------------------------------------------------------------------------- + +// serializeSizeVLQ returns the number of bytes it would take to serialize the +// passed number as a variable-length quantity according to the format described +// above. +func serializeSizeVLQ(n uint64) int { + size := 1 + for ; n > 0x7f; n = (n >> 7) - 1 { + size++ + } + + return size +} + +// putVLQ serializes the provided number to a variable-length quantity according +// to the format described above and returns the number of bytes of the encoded +// value. The result is placed directly into the passed byte slice which must +// be at least large enough to handle the number of bytes returned by the +// serializeSizeVLQ function or it will panic. +func putVLQ(target []byte, n uint64) int { + offset := 0 + for ; ; offset++ { + // The high bit is set when another byte follows. + highBitMask := byte(0x80) + if offset == 0 { + highBitMask = 0x00 + } + + target[offset] = byte(n&0x7f) | highBitMask + if n <= 0x7f { + break + } + n = (n >> 7) - 1 + } + + // Reverse the bytes so it is MSB-encoded. + for i, j := 0, offset; i < j; i, j = i+1, j-1 { + target[i], target[j] = target[j], target[i] + } + + return offset + 1 +} + +// deserializeVLQ deserializes the provided variable-length quantity according +// to the format described above. It also returns the number of bytes +// deserialized. +func deserializeVLQ(serialized []byte) (uint64, int) { + var n uint64 + var size int + for _, val := range serialized { + size++ + n = (n << 7) | uint64(val&0x7f) + if val&0x80 != 0x80 { + break + } + n++ + } + + return n, size +} + +// ----------------------------------------------------------------------------- +// In order to reduce the size of stored scripts, a domain specific compression +// algorithm is used which recognizes standard scripts and stores them using +// less bytes than the original script. The compression algorithm used here was +// obtained from Bitcoin Core, so all credits for the algorithm go to it. +// +// The general serialized format is: +// +//