LBRYCommunity/lbcd
1
0
Fork 0
Code Issues 17 Pull requests 4 Projects Releases 28 Packages Wiki Activity

Merge pull request #1045 from davecgh/multi_utxoentry_outpoints

Browse source 
multi: Rework utxoset/view to use outpoints.
This commit is contained in:
Olaoluwa Osuntokun 2018-05-30 19:59:44 -07:00 committed by GitHub
commit 86fed78113
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1533 additions and 1363 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
blockchain/chain.go
View file

@ -399,7 +399,7 @@ func (b *BlockChain) calcSequenceLock(node *blockNode, tx *btcutil.Tx, utxoView
nextHeight := node.height + 1
for txInIndex, txIn := range mTx.TxIn {
utxo := utxoView.LookupEntry(&txIn.PreviousOutPoint.Hash)
utxo := utxoView.LookupEntry(txIn.PreviousOutPoint)
if utxo == nil {
str := fmt.Sprintf("output %v referenced from "+
"transaction %s:%d either does not exist or "+
@ -848,7 +848,7 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List) error
// journal.
var stxos []spentTxOut
err = b.db.View(func(dbTx database.Tx) error {
stxos, err = dbFetchSpendJournalEntry(dbTx, block, view)
stxos, err = dbFetchSpendJournalEntry(dbTx, block)
return err
})
if err != nil {
@ -859,7 +859,7 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List) error
detachBlocks = append(detachBlocks, block)
detachSpentTxOuts = append(detachSpentTxOuts, stxos)
err = view.disconnectTransactions(block, stxos)
err = view.disconnectTransactions(b.db, block, stxos)
if err != nil {
return err
}
@ -961,7 +961,8 @@ func (b *BlockChain) reorganizeChain(detachNodes, attachNodes *list.List) error
// Update the view to unspend all of the spent txos and remove
// the utxos created by the block.
err = view.disconnectTransactions(block, detachSpentTxOuts[i])
err = view.disconnectTransactions(b.db, block,
detachSpentTxOuts[i])
if err != nil {
return err
}
@ -1702,6 +1703,11 @@ func New(config *Config) (*BlockChain, error) {
return nil, err
}
// Perform any upgrades to the various chain-specific buckets as needed.
if err := b.maybeUpgradeDbBuckets(config.Interrupt); err != nil {
return nil, err
}
// Initialize and catch up all of the currently active optional indexes
// as needed.
if config.IndexManager != nil {

713
blockchain/chainio.go
View file

@ -9,7 +9,7 @@ import (
"encoding/binary"
"fmt"
"math/big"
"sort"
"sync"
"time"
"github.com/btcsuite/btcd/chaincfg/chainhash"
@ -23,6 +23,15 @@ const (
// constant from wire and is only provided here for convenience since
// wire.MaxBlockHeaderPayload is quite long.
blockHdrSize = wire.MaxBlockHeaderPayload
// latestUtxoSetBucketVersion is the current version of the utxo set
// bucket that is used to track all unspent outputs.
latestUtxoSetBucketVersion = 2
// latestSpendJournalBucketVersion is the current version of the spend
// journal bucket that is used to track all spent transactions for use
// in reorgs.
latestSpendJournalBucketVersion = 1
)
var (
@ -42,13 +51,21 @@ var (
// chain state.
chainStateKeyName = []byte("chainstate")
// spendJournalVersionKeyName is the name of the db key used to store
// the version of the spend journal currently in the database.
spendJournalVersionKeyName = []byte("spendjournalversion")
// spendJournalBucketName is the name of the db bucket used to house
// transactions outputs that are spent in each block.
spendJournalBucketName = []byte("spendjournal")
// utxoSetVersionKeyName is the name of the db key used to store the
// version of the utxo set currently in the database.
utxoSetVersionKeyName = []byte("utxosetversion")
// utxoSetBucketName is the name of the db bucket used to house the
// unspent transaction output set.
utxoSetBucketName = []byte("utxoset")
utxoSetBucketName = []byte("utxosetv2")
// byteOrder is the preferred byte order used for serializing numeric
// fields for storage in the database.
@ -94,6 +111,45 @@ func isDbBucketNotFoundErr(err error) bool {
return ok && dbErr.ErrorCode == database.ErrBucketNotFound
}
// dbFetchVersion fetches an individual version with the given key from the
// metadata bucket. It is primarily used to track versions on entities such as
// buckets. It returns zero if the provided key does not exist.
func dbFetchVersion(dbTx database.Tx, key []byte) uint32 {
serialized := dbTx.Metadata().Get(key)
if serialized == nil {
return 0
}
return byteOrder.Uint32(serialized[:])
}
// dbPutVersion uses an existing database transaction to update the provided
// key in the metadata bucket to the given version. It is primarily used to
// track versions on entities such as buckets.
func dbPutVersion(dbTx database.Tx, key []byte, version uint32) error {
var serialized [4]byte
byteOrder.PutUint32(serialized[:], version)
return dbTx.Metadata().Put(key, serialized[:])
}
// dbFetchOrCreateVersion uses an existing database transaction to attempt to
// fetch the provided key from the metadata bucket as a version and in the case
// it doesn't exist, it adds the entry with the provided default version and
// returns that. This is useful during upgrades to automatically handle loading
// and adding version keys as necessary.
func dbFetchOrCreateVersion(dbTx database.Tx, key []byte, defaultVersion uint32) (uint32, error) {
version := dbFetchVersion(dbTx, key)
if version == 0 {
version = defaultVersion
err := dbPutVersion(dbTx, key, version)
if err != nil {
return 0, err
}
}
return version, nil
}
// -----------------------------------------------------------------------------
// The transaction spend journal consists of an entry for each block connected
// to the main chain which contains the transaction outputs the block spends
@ -110,18 +166,23 @@ func isDbBucketNotFoundErr(err error) bool {
//
// NOTE: This format is NOT self describing. The additional details such as
// the number of entries (transaction inputs) are expected to come from the
// block itself and the utxo set. The rationale in doing this is to save a
// significant amount of space. This is also the reason the spent outputs are
// serialized in the reverse order they are spent because later transactions
// are allowed to spend outputs from earlier ones in the same block.
// block itself and the utxo set (for legacy entries). The rationale in doing
// this is to save space. This is also the reason the spent outputs are
// serialized in the reverse order they are spent because later transactions are
// allowed to spend outputs from earlier ones in the same block.
//
// The reserved field below used to keep track of the version of the containing
// transaction when the height in the header code was non-zero, however the
// height is always non-zero now, but keeping the extra reserved field allows
// backwards compatibility.
//
// The serialized format is:
//
// [<header code><version><compressed txout>],...
// [<header code><reserved><compressed txout>],...
//
// Field Type Size
// header code VLQ variable
// version VLQ variable
// reserved byte 1
// compressed txout
// compressed amount VLQ variable
// compressed script []byte variable
@ -130,23 +191,17 @@ func isDbBucketNotFoundErr(err error) bool {
// bit 0 - containing transaction is a coinbase
// bits 1-x - height of the block that contains the spent txout
//
// NOTE: The header code and version are only encoded when the spent txout was
// the final unspent output of the containing transaction. Otherwise, the
// header code will be 0 and the version is not serialized at all. This is
// done because that information is only needed when the utxo set no longer
// has it.
//
// Example 1:
// From block 170 in main blockchain.
//
// 1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c
// 1300320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c
// <><><------------------------------------------------------------------>
// | | |
// | version compressed txout
// | reserved compressed txout
// header code
//
// - header code: 0x13 (coinbase, height 9)
// - transaction version: 1
// - reserved: 0x00
// - compressed txout 0:
// - 0x32: VLQ-encoded compressed amount for 5000000000 (50 BTC)
// - 0x05: special script type pay-to-pubkey
@ -155,22 +210,22 @@ func isDbBucketNotFoundErr(err error) bool {
// Example 2:
// Adapted from block 100025 in main blockchain.
//
// 0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e868b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec
// <><----------------------------------------------><----><><---------------------------------------------->
// | | | | |
// | compressed txout | version compressed txout
// header code header code
// 8b99700091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e868b99700086c64700b2fb57eadf61e106a100a7445a8c3f67898841ec
// <----><><----------------------------------------------><----><><---------------------------------------------->
// | | | | | |
// | reserved compressed txout | reserved compressed txout
// header code header code
//
// - Last spent output:
// - header code: 0x00 (was not the final unspent output for containing tx)
// - transaction version: Nothing since header code is 0
// - header code: 0x8b9970 (not coinbase, height 100024)
// - reserved: 0x00
// - compressed txout:
// - 0x91f20f: VLQ-encoded compressed amount for 34405000000 (344.05 BTC)
// - 0x00: special script type pay-to-pubkey-hash
// - 0x6e...86: pubkey hash
// - Second to last spent output:
// - header code: 0x8b9970 (not coinbase, height 100024)
// - transaction version: 1
// - reserved: 0x00
// - compressed txout:
// - 0x86c647: VLQ-encoded compressed amount for 13761000000 (137.61 BTC)
// - 0x00: special script type pay-to-pubkey-hash
@ -185,25 +240,15 @@ func isDbBucketNotFoundErr(err error) bool {
// when this spent txout is spending the last unspent output of the containing
// transaction.
type spentTxOut struct {
compressed bool // The amount and public key script are compressed.
version int32 // The version of creating tx.
amount int64 // The amount of the output.
pkScript []byte // The public key script for the output.
// These fields are only set when this is spending the final output of
// the creating tx.
height int32 // Height of the the block containing the creating tx.
isCoinBase bool // Whether creating tx is a coinbase.
height int32 // Height of the the block containing the creating tx.
isCoinBase bool // Whether creating tx is a coinbase.
}
// spentTxOutHeaderCode returns the calculated header code to be used when
// serializing the provided stxo entry.
func spentTxOutHeaderCode(stxo *spentTxOut) uint64 {
// The header code is 0 when there is no height set for the stxo.
if stxo.height == 0 {
return 0
}
// As described in the serialization format comments, the header code
// encodes the height shifted over one bit and the coinbase flag in the
// lowest bit.
@ -218,13 +263,14 @@ func spentTxOutHeaderCode(stxo *spentTxOut) uint64 {
// spentTxOutSerializeSize returns the number of bytes it would take to
// serialize the passed stxo according to the format described above.
func spentTxOutSerializeSize(stxo *spentTxOut) int {
headerCode := spentTxOutHeaderCode(stxo)
size := serializeSizeVLQ(headerCode)
if headerCode != 0 {
size += serializeSizeVLQ(uint64(stxo.version))
size := serializeSizeVLQ(spentTxOutHeaderCode(stxo))
if stxo.height > 0 {
// The legacy v1 spend journal format conditionally tracked the
// containing transaction version when the height was non-zero,
// so this is required for backwards compat.
size += serializeSizeVLQ(0)
}
return size + compressedTxOutSize(uint64(stxo.amount), stxo.pkScript,
stxo.version, stxo.compressed)
return size + compressedTxOutSize(uint64(stxo.amount), stxo.pkScript)
}
// putSpentTxOut serializes the passed stxo according to the format described
@ -234,26 +280,20 @@ func spentTxOutSerializeSize(stxo *spentTxOut) int {
func putSpentTxOut(target []byte, stxo *spentTxOut) int {
headerCode := spentTxOutHeaderCode(stxo)
offset := putVLQ(target, headerCode)
if headerCode != 0 {
offset += putVLQ(target[offset:], uint64(stxo.version))
if stxo.height > 0 {
// The legacy v1 spend journal format conditionally tracked the
// containing transaction version when the height was non-zero,
// so this is required for backwards compat.
offset += putVLQ(target[offset:], 0)
}
return offset + putCompressedTxOut(target[offset:], uint64(stxo.amount),
stxo.pkScript, stxo.version, stxo.compressed)
stxo.pkScript)
}
// decodeSpentTxOut decodes the passed serialized stxo entry, possibly followed
// by other data, into the passed stxo struct. It returns the number of bytes
// read.
//
// Since the serialized stxo entry does not contain the height, version, or
// coinbase flag of the containing transaction when it still has utxos, the
// caller is responsible for passing in the containing transaction version in
// that case. The provided version is ignore when it is serialized as a part of
// the stxo.
//
// An error will be returned if the version is not serialized as a part of the
// stxo and is also not provided to the function.
func decodeSpentTxOut(serialized []byte, stxo *spentTxOut, txVersion int32) (int, error) {
func decodeSpentTxOut(serialized []byte, stxo *spentTxOut) (int, error) {
// Ensure there are bytes to decode.
if len(serialized) == 0 {
return 0, errDeserialize("no serialized bytes")
@ -266,47 +306,34 @@ func decodeSpentTxOut(serialized []byte, stxo *spentTxOut, txVersion int32) (int
"header code")
}
// Decode the header code and deserialize the containing transaction
// version if needed.
// Decode the header code.
//
// Bit 0 indicates containing transaction is a coinbase.
// Bits 1-x encode height of containing transaction.
if code != 0 {
version, bytesRead := deserializeVLQ(serialized[offset:])
stxo.isCoinBase = code&0x01 != 0
stxo.height = int32(code >> 1)
if stxo.height > 0 {
// The legacy v1 spend journal format conditionally tracked the
// containing transaction version when the height was non-zero,
// so this is required for backwards compat.
_, bytesRead := deserializeVLQ(serialized[offset:])
offset += bytesRead
if offset >= len(serialized) {
return offset, errDeserialize("unexpected end of data " +
"after version")
"after reserved")
}
stxo.isCoinBase = code&0x01 != 0
stxo.height = int32(code >> 1)
stxo.version = int32(version)
} else {
// Ensure a tx version was specified if the stxo did not encode
// it. This should never happen unless there is database
// corruption or this function is being called without the
// proper state.
if txVersion == -1 {
return offset, AssertError("decodeSpentTxOut called " +
"without a containing tx version when the " +
"serialized stxo that does not encode the " +
"version")
}
stxo.version = txVersion
}
// Decode the compressed txout.
compAmount, compScript, bytesRead, err := decodeCompressedTxOut(
serialized[offset:], stxo.version)
amount, pkScript, bytesRead, err := decodeCompressedTxOut(
serialized[offset:])
offset += bytesRead
if err != nil {
return offset, errDeserialize(fmt.Sprintf("unable to decode "+
"txout: %v", err))
}
stxo.amount = int64(compAmount)
stxo.pkScript = compScript
stxo.compressed = true
stxo.amount = int64(amount)
stxo.pkScript = pkScript
return offset, nil
}
@ -315,9 +342,8 @@ func decodeSpentTxOut(serialized []byte, stxo *spentTxOut, txVersion int32) (int
//
// Since the serialization format is not self describing, as noted in the
// format comments, this function also requires the transactions that spend the
// txouts and a utxo view that contains any remaining existing utxos in the
// transactions referenced by the inputs to the passed transasctions.
func deserializeSpendJournalEntry(serialized []byte, txns []*wire.MsgTx, view *UtxoViewpoint) ([]spentTxOut, error) {
// txouts.
func deserializeSpendJournalEntry(serialized []byte, txns []*wire.MsgTx) ([]spentTxOut, error) {
// Calculate the total number of stxos.
var numStxos int
for _, tx := range txns {
@ -341,7 +367,6 @@ func deserializeSpendJournalEntry(serialized []byte, txns []*wire.MsgTx, view *U
// Loop backwards through all transactions so everything is read in
// reverse order to match the serialization order.
stxoIdx := numStxos - 1
stxoInFlight := make(map[chainhash.Hash]int)
offset := 0
stxos := make([]spentTxOut, numStxos)
for txIdx := len(txns) - 1; txIdx > -1; txIdx-- {
@ -354,36 +379,7 @@ func deserializeSpendJournalEntry(serialized []byte, txns []*wire.MsgTx, view *U
stxo := &stxos[stxoIdx]
stxoIdx--
// Get the transaction version for the stxo based on
// whether or not it should be serialized as a part of
// the stxo. Recall that it is only serialized when the
// stxo spends the final utxo of a transaction. Since
// they are deserialized in reverse order, this means
// the first time an entry for a given containing tx is
// encountered that is not already in the utxo view it
// must have been the final spend and thus the extra
// data will be serialized with the stxo. Otherwise,
// the version must be pulled from the utxo entry.
//
// Since the view is not actually modified as the stxos
// are read here and it's possible later entries
// reference earlier ones, an inflight map is maintained
// to detect this case and pull the tx version from the
// entry that contains the version information as just
// described.
txVersion := int32(-1)
originHash := &txIn.PreviousOutPoint.Hash
entry := view.LookupEntry(originHash)
if entry != nil {
txVersion = entry.Version()
} else if idx, ok := stxoInFlight[*originHash]; ok {
txVersion = stxos[idx].version
} else {
stxoInFlight[*originHash] = stxoIdx + 1
}
n, err := decodeSpentTxOut(serialized[offset:], stxo,
txVersion)
n, err := decodeSpentTxOut(serialized[offset:], stxo)
offset += n
if err != nil {
return nil, errDeserialize(fmt.Sprintf("unable "+
@ -420,17 +416,18 @@ func serializeSpendJournalEntry(stxos []spentTxOut) []byte {
return serialized
}
// dbFetchSpendJournalEntry fetches the spend journal entry for the passed
// block and deserializes it into a slice of spent txout entries. The provided
// view MUST have the utxos referenced by all of the transactions available for
// the passed block since that information is required to reconstruct the spent
// txouts.
func dbFetchSpendJournalEntry(dbTx database.Tx, block *btcutil.Block, view *UtxoViewpoint) ([]spentTxOut, error) {
// dbFetchSpendJournalEntry fetches the spend journal entry for the passed block
// and deserializes it into a slice of spent txout entries.
//
// NOTE: Legacy entries will not have the coinbase flag or height set unless it
// was the final output spend in the containing transaction. It is up to the
// caller to handle this properly by looking the information up in the utxo set.
func dbFetchSpendJournalEntry(dbTx database.Tx, block *btcutil.Block) ([]spentTxOut, error) {
// Exclude the coinbase transaction since it can't spend anything.
spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName)
serialized := spendBucket.Get(block.Hash()[:])
blockTxns := block.MsgBlock().Transactions[1:]
stxos, err := deserializeSpendJournalEntry(serialized, blockTxns, view)
stxos, err := deserializeSpendJournalEntry(serialized, blockTxns)
if err != nil {
// Ensure any deserialization errors are returned as database
// corruption errors.
@ -468,219 +465,161 @@ func dbRemoveSpendJournalEntry(dbTx database.Tx, blockHash *chainhash.Hash) erro
// -----------------------------------------------------------------------------
// The unspent transaction output (utxo) set consists of an entry for each
// transaction which contains a utxo serialized using a format that is highly
// optimized to reduce space using domain specific compression algorithms. This
// format is a slightly modified version of the format used in Bitcoin Core.
// unspent output using a format that is optimized to reduce space using domain
// specific compression algorithms. This format is a slightly modified version
// of the format used in Bitcoin Core.
//
// The serialized format is:
// Each entry is keyed by an outpoint as specified below. It is important to
// note that the key encoding uses a VLQ, which employs an MSB encoding so
// iteration of utxos when doing byte-wise comparisons will produce them in
// order.
//
// <version><height><header code><unspentness bitmap>[<compressed txouts>,...]
// The serialized key format is:
// <hash><output index>
//
// Field Type Size
// hash chainhash.Hash chainhash.HashSize
// output index VLQ variable
//
// The serialized value format is:
//
// <header code><compressed txout>
//
// Field Type Size
// version VLQ variable
// block height VLQ variable
// header code VLQ variable
// unspentness bitmap []byte variable
// compressed txouts
// compressed txout
// compressed amount VLQ variable
// compressed script []byte variable
//
// The serialized header code format is:
// bit 0 - containing transaction is a coinbase
// bit 1 - output zero is unspent
// bit 2 - output one is unspent
// bits 3-x - number of bytes in unspentness bitmap. When both bits 1 and 2
// are unset, it encodes N-1 since there must be at least one unspent
// output.
//
// The rationale for the header code scheme is as follows:
// - Transactions which only pay to a single output and a change output are
// extremely common, thus an extra byte for the unspentness bitmap can be
// avoided for them by encoding those two outputs in the low order bits.
// - Given it is encoded as a VLQ which can encode values up to 127 with a
// single byte, that leaves 4 bits to represent the number of bytes in the
// unspentness bitmap while still only consuming a single byte for the
// header code. In other words, an unspentness bitmap with up to 120
// transaction outputs can be encoded with a single-byte header code.
// This covers the vast majority of transactions.
// - Encoding N-1 bytes when both bits 1 and 2 are unset allows an additional
// 8 outpoints to be encoded before causing the header code to require an
// additional byte.
// bits 1-x - height of the block that contains the unspent txout
//
// Example 1:
// From tx in main blockchain:
// Blk 1, 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098
// Blk 1, 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098:0
//
// 010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52
// <><><><------------------------------------------------------------------>
// | | \--------\ |
// | height | compressed txout 0
// version header code
// 03320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52
// <><------------------------------------------------------------------>
// | |
// header code compressed txout
//
// - version: 1
// - height: 1
// - header code: 0x03 (coinbase, output zero unspent, 0 bytes of unspentness)
// - unspentness: Nothing since it is zero bytes
// - compressed txout 0:
// - header code: 0x03 (coinbase, height 1)
// - compressed txout:
// - 0x32: VLQ-encoded compressed amount for 5000000000 (50 BTC)
// - 0x04: special script type pay-to-pubkey
// - 0x96...52: x-coordinate of the pubkey
//
// Example 2:
// From tx in main blockchain:
// Blk 113931, 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f
// Blk 113931, 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f:2
//
// 0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58
// <><----><><><------------------------------------------><-------------------------------------------->
// | | | \-------------------\ | |
// version | \--------\ unspentness | compressed txout 2
// height header code compressed txout 0
// 8cf316800900b8025be1b3efc63b0ad48e7f9f10e87544528d58
// <----><------------------------------------------>
// | |
// header code compressed txout
//
// - version: 1
// - height: 113931
// - header code: 0x0a (output zero unspent, 1 byte in unspentness bitmap)
// - unspentness: [0x01] (bit 0 is set, so output 0+2 = 2 is unspent)
// NOTE: It's +2 since the first two outputs are encoded in the header code
// - compressed txout 0:
// - 0x12: VLQ-encoded compressed amount for 20000000 (0.2 BTC)
// - 0x00: special script type pay-to-pubkey-hash
// - 0xe2...8a: pubkey hash
// - compressed txout 2:
// - header code: 0x8cf316 (not coinbase, height 113931)
// - compressed txout:
// - 0x8009: VLQ-encoded compressed amount for 15000000 (0.15 BTC)
// - 0x00: special script type pay-to-pubkey-hash
// - 0xb8...58: pubkey hash
//
// Example 3:
// From tx in main blockchain:
// Blk 338156, 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620
// Blk 338156, 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620:22
//
// 0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6
// <><----><><----><-------------------------------------------------->
// | | | \-----------------\ |
// version | \--------\ unspentness |
// height header code compressed txout 22
// a8a2588ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6
// <----><-------------------------------------------------->
// | |
// header code compressed txout
//
// - version: 1
// - height: 338156
// - header code: 0x10 (2+1 = 3 bytes in unspentness bitmap)
// NOTE: It's +1 since neither bit 1 nor 2 are set, so N-1 is encoded.
// - unspentness: [0x00 0x00 0x10] (bit 20 is set, so output 20+2 = 22 is unspent)
// NOTE: It's +2 since the first two outputs are encoded in the header code
// - compressed txout 22:
// - header code: 0xa8a258 (not coinbase, height 338156)
// - compressed txout:
// - 0x8ba5b9e763: VLQ-encoded compressed amount for 366875659 (3.66875659 BTC)
// - 0x01: special script type pay-to-script-hash
// - 0x1d...e6: script hash
// -----------------------------------------------------------------------------
// maxUint32VLQSerializeSize is the maximum number of bytes a max uint32 takes
// to serialize as a VLQ.
var maxUint32VLQSerializeSize = serializeSizeVLQ(1<<32 - 1)
// outpointKeyPool defines a concurrent safe free list of byte slices used to
// provide temporary buffers for outpoint database keys.
var outpointKeyPool = sync.Pool{
New: func() interface{} {
b := make([]byte, chainhash.HashSize+maxUint32VLQSerializeSize)
return &b // Pointer to slice to avoid boxing alloc.
},
}
// outpointKey returns a key suitable for use as a database key in the utxo set
// while making use of a free list. A new buffer is allocated if there are not
// already any available on the free list. The returned byte slice should be
// returned to the free list by using the recycleOutpointKey function when the
// caller is done with it _unless_ the slice will need to live for longer than
// the caller can calculate such as when used to write to the database.
func outpointKey(outpoint wire.OutPoint) *[]byte {
// A VLQ employs an MSB encoding, so they are useful not only to reduce
// the amount of storage space, but also so iteration of utxos when
// doing byte-wise comparisons will produce them in order.
key := outpointKeyPool.Get().(*[]byte)
idx := uint64(outpoint.Index)
*key = (*key)[:chainhash.HashSize+serializeSizeVLQ(idx)]
copy(*key, outpoint.Hash[:])
putVLQ((*key)[chainhash.HashSize:], idx)
return key
}
// recycleOutpointKey puts the provided byte slice, which should have been
// obtained via the outpointKey function, back on the free list.
func recycleOutpointKey(key *[]byte) {
outpointKeyPool.Put(key)
}
// utxoEntryHeaderCode returns the calculated header code to be used when
// serializing the provided utxo entry and the number of bytes needed to encode
// the unspentness bitmap.
func utxoEntryHeaderCode(entry *UtxoEntry, highestOutputIndex uint32) (uint64, int, error) {
// The first two outputs are encoded separately, so offset the index
// accordingly to calculate the correct number of bytes needed to encode
// up to the highest unspent output index.
numBitmapBytes := int((highestOutputIndex + 6) / 8)
// As previously described, one less than the number of bytes is encoded
// when both output 0 and 1 are spent because there must be at least one
// unspent output. Adjust the number of bytes to encode accordingly and
// encode the value by shifting it over 3 bits.
output0Unspent := !entry.IsOutputSpent(0)
output1Unspent := !entry.IsOutputSpent(1)
var numBitmapBytesAdjustment int
if !output0Unspent && !output1Unspent {
if numBitmapBytes == 0 {
return 0, 0, AssertError("attempt to serialize utxo " +
"header for fully spent transaction")
}
numBitmapBytesAdjustment = 1
}
headerCode := uint64(numBitmapBytes-numBitmapBytesAdjustment) << 3
// Set the coinbase, output 0, and output 1 bits in the header code
// accordingly.
if entry.isCoinBase {
headerCode |= 0x01 // bit 0
}
if output0Unspent {
headerCode |= 0x02 // bit 1
}
if output1Unspent {
headerCode |= 0x04 // bit 2
// serializing the provided utxo entry.
func utxoEntryHeaderCode(entry *UtxoEntry) (uint64, error) {
if entry.IsSpent() {
return 0, AssertError("attempt to serialize spent utxo header")
}
return headerCode, numBitmapBytes, nil
// As described in the serialization format comments, the header code
// encodes the height shifted over one bit and the coinbase flag in the
// lowest bit.
headerCode := uint64(entry.BlockHeight()) << 1
if entry.IsCoinBase() {
headerCode |= 0x01
}
return headerCode, nil
}
// serializeUtxoEntry returns the entry serialized to a format that is suitable
// for long-term storage. The format is described in detail above.
func serializeUtxoEntry(entry *UtxoEntry) ([]byte, error) {
// Fully spent entries have no serialization.
if entry.IsFullySpent() {
// Spent outputs have no serialization.
if entry.IsSpent() {
return nil, nil
}
// Determine the output order by sorting the sparse output index keys.
outputOrder := make([]int, 0, len(entry.sparseOutputs))
for outputIndex := range entry.sparseOutputs {
outputOrder = append(outputOrder, int(outputIndex))
}
sort.Ints(outputOrder)
// Encode the header code and determine the number of bytes the
// unspentness bitmap needs.
highIndex := uint32(outputOrder[len(outputOrder)-1])
headerCode, numBitmapBytes, err := utxoEntryHeaderCode(entry, highIndex)
// Encode the header code.
headerCode, err := utxoEntryHeaderCode(entry)
if err != nil {
return nil, err
}
// Calculate the size needed to serialize the entry.
size := serializeSizeVLQ(uint64(entry.version)) +
serializeSizeVLQ(uint64(entry.blockHeight)) +
serializeSizeVLQ(headerCode) + numBitmapBytes
for _, outputIndex := range outputOrder {
out := entry.sparseOutputs[uint32(outputIndex)]
if out.spent {
continue
}
size += compressedTxOutSize(uint64(out.amount), out.pkScript,
entry.version, out.compressed)
}
size := serializeSizeVLQ(headerCode) +
compressedTxOutSize(uint64(entry.Amount()), entry.PkScript())
// Serialize the version, block height of the containing transaction,
// and header code.
// Serialize the header code followed by the compressed unspent
// transaction output.
serialized := make([]byte, size)
offset := putVLQ(serialized, uint64(entry.version))
offset += putVLQ(serialized[offset:], uint64(entry.blockHeight))
offset += putVLQ(serialized[offset:], headerCode)
// Serialize the unspentness bitmap.
for i := uint32(0); i < uint32(numBitmapBytes); i++ {
unspentBits := byte(0)
for j := uint32(0); j < 8; j++ {
// The first 2 outputs are encoded via the header code,
// so adjust the output index accordingly.
if !entry.IsOutputSpent(2 + i*8 + j) {
unspentBits |= 1 << uint8(j)
}
}
serialized[offset] = unspentBits
offset++
}
// Serialize the compressed unspent transaction outputs. Outputs that
// are already compressed are serialized without modifications.
for _, outputIndex := range outputOrder {
out := entry.sparseOutputs[uint32(outputIndex)]
if out.spent {
continue
}
offset += putCompressedTxOut(serialized[offset:],
uint64(out.amount), out.pkScript, entry.version,
out.compressed)
}
offset := putVLQ(serialized, headerCode)
offset += putCompressedTxOut(serialized[offset:], uint64(entry.Amount()),
entry.PkScript())
return serialized, nil
}
@ -689,23 +628,8 @@ func serializeUtxoEntry(entry *UtxoEntry) ([]byte, error) {
// slice into a new UtxoEntry using a format that is suitable for long-term
// storage. The format is described in detail above.
func deserializeUtxoEntry(serialized []byte) (*UtxoEntry, error) {
// Deserialize the version.
version, bytesRead := deserializeVLQ(serialized)
offset := bytesRead
if offset >= len(serialized) {
return nil, errDeserialize("unexpected end of data after version")
}
// Deserialize the block height.
blockHeight, bytesRead := deserializeVLQ(serialized[offset:])
offset += bytesRead
if offset >= len(serialized) {
return nil, errDeserialize("unexpected end of data after height")
}
// Deserialize the header code.
code, bytesRead := deserializeVLQ(serialized[offset:])
offset += bytesRead
code, offset := deserializeVLQ(serialized)
if offset >= len(serialized) {
return nil, errDeserialize("unexpected end of data after header")
}
@ -713,101 +637,83 @@ func deserializeUtxoEntry(serialized []byte) (*UtxoEntry, error) {
// Decode the header code.
//
// Bit 0 indicates whether the containing transaction is a coinbase.
// Bit 1 indicates output 0 is unspent.
// Bit 2 indicates output 1 is unspent.
// Bits 3-x encodes the number of non-zero unspentness bitmap bytes that
// follow. When both output 0 and 1 are spent, it encodes N-1.
// Bits 1-x encode height of containing transaction.
isCoinBase := code&0x01 != 0
output0Unspent := code&0x02 != 0
output1Unspent := code&0x04 != 0
numBitmapBytes := code >> 3
if !output0Unspent && !output1Unspent {
numBitmapBytes++
blockHeight := int32(code >> 1)
// Decode the compressed unspent transaction output.
amount, pkScript, _, err := decodeCompressedTxOut(serialized[offset:])
if err != nil {
return nil, errDeserialize(fmt.Sprintf("unable to decode "+
"utxo: %v", err))
}
// Ensure there are enough bytes left to deserialize the unspentness
// bitmap.
if uint64(len(serialized[offset:])) < numBitmapBytes {
return nil, errDeserialize("unexpected end of data for " +
"unspentness bitmap")
entry := &UtxoEntry{
amount: int64(amount),
pkScript: pkScript,
blockHeight: blockHeight,
packedFlags: 0,
}
// Create a new utxo entry with the details deserialized above to house
// all of the utxos.
entry := newUtxoEntry(int32(version), isCoinBase, int32(blockHeight))
// Add sparse output for unspent outputs 0 and 1 as needed based on the
// details provided by the header code.
var outputIndexes []uint32
if output0Unspent {
outputIndexes = append(outputIndexes, 0)
}
if output1Unspent {
outputIndexes = append(outputIndexes, 1)
}
// Decode the unspentness bitmap adding a sparse output for each unspent
// output.
for i := uint32(0); i < uint32(numBitmapBytes); i++ {
unspentBits := serialized[offset]
for j := uint32(0); j < 8; j++ {
if unspentBits&0x01 != 0 {
// The first 2 outputs are encoded via the
// header code, so adjust the output number
// accordingly.
outputNum := 2 + i*8 + j
outputIndexes = append(outputIndexes, outputNum)
}
unspentBits >>= 1
}
offset++
}
// Decode and add all of the utxos.
for i, outputIndex := range outputIndexes {
// Decode the next utxo. The script and amount fields of the
// utxo output are left compressed so decompression can be
// avoided on those that are not accessed. This is done since
// it is quite common for a redeeming transaction to only
// reference a single utxo from a referenced transaction.
compAmount, compScript, bytesRead, err := decodeCompressedTxOut(
serialized[offset:], int32(version))
if err != nil {
return nil, errDeserialize(fmt.Sprintf("unable to "+
"decode utxo at index %d: %v", i, err))
}
offset += bytesRead
entry.sparseOutputs[outputIndex] = &utxoOutput{
spent: false,
compressed: true,
pkScript: compScript,
amount: int64(compAmount),
}
if isCoinBase {
entry.packedFlags |= tfCoinBase
}
return entry, nil
}
// dbFetchUtxoEntry uses an existing database transaction to fetch all unspent
// outputs for the provided Bitcoin transaction hash from the utxo set.
// dbFetchUtxoEntryByHash attempts to find and fetch a utxo for the given hash.
// It uses a cursor and seek to try and do this as efficiently as possible.
//
// When there is no entry for the provided hash, nil will be returned for the
// When there are no entries for the provided hash, nil will be returned for the
// both the entry and the error.
func dbFetchUtxoEntry(dbTx database.Tx, hash *chainhash.Hash) (*UtxoEntry, error) {
func dbFetchUtxoEntryByHash(dbTx database.Tx, hash *chainhash.Hash) (*UtxoEntry, error) {
// Attempt to find an entry by seeking for the hash along with a zero
// index. Due to the fact the keys are serialized as <hash><index>,
// where the index uses an MSB encoding, if there are any entries for
// the hash at all, one will be found.
cursor := dbTx.Metadata().Bucket(utxoSetBucketName).Cursor()
key := outpointKey(wire.OutPoint{Hash: *hash, Index: 0})
ok := cursor.Seek(*key)
recycleOutpointKey(key)
if !ok {
return nil, nil
}
// An entry was found, but it could just be an entry with the next
// highest hash after the requested one, so make sure the hashes
// actually match.
cursorKey := cursor.Key()
if len(cursorKey) < chainhash.HashSize {
return nil, nil
}
if !bytes.Equal(hash[:], cursorKey[:chainhash.HashSize]) {
return nil, nil
}
return deserializeUtxoEntry(cursor.Value())
}
// dbFetchUtxoEntry uses an existing database transaction to fetch the specified
// transaction output from the utxo set.
//
// When there is no entry for the provided output, nil will be returned for both
// the entry and the error.
func dbFetchUtxoEntry(dbTx database.Tx, outpoint wire.OutPoint) (*UtxoEntry, error) {
// Fetch the unspent transaction output information for the passed
// transaction hash. Return now when there is no entry.
// transaction output. Return now when there is no entry.
key := outpointKey(outpoint)
utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName)
serializedUtxo := utxoBucket.Get(hash[:])
serializedUtxo := utxoBucket.Get(*key)
recycleOutpointKey(key)
if serializedUtxo == nil {
return nil, nil
}
// A non-nil zero-length entry means there is an entry in the database
// for a fully spent transaction which should never be the case.
// for a spent transaction output which should never be the case.
if len(serializedUtxo) == 0 {
return nil, AssertError(fmt.Sprintf("database contains entry "+
"for fully spent tx %v", hash))
"for spent tx output %v", outpoint))
}
// Deserialize the utxo entry and return it.
@ -819,7 +725,7 @@ func dbFetchUtxoEntry(dbTx database.Tx, hash *chainhash.Hash) (*UtxoEntry, error
return nil, database.Error{
ErrorCode: database.ErrCorruption,
Description: fmt.Sprintf("corrupt utxo entry "+
"for %v: %v", hash, err),
"for %v: %v", outpoint, err),
}
}
@ -835,36 +741,35 @@ func dbFetchUtxoEntry(dbTx database.Tx, hash *chainhash.Hash) (*UtxoEntry, error
// to the database.
func dbPutUtxoView(dbTx database.Tx, view *UtxoViewpoint) error {
utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName)
for txHashIter, entry := range view.entries {
for outpoint, entry := range view.entries {
// No need to update the database if the entry was not modified.
if entry == nil || !entry.modified {
if entry == nil || !entry.isModified() {
continue
}
// Serialize the utxo entry without any entries that have been
// spent.
serialized, err := serializeUtxoEntry(entry)
if err != nil {
return err
}
// Make a copy of the hash because the iterator changes on each
// loop iteration and thus slicing it directly would cause the
// data to change out from under the put/delete funcs below.
txHash := txHashIter
// Remove the utxo entry if it is now fully spent.
if serialized == nil {
if err := utxoBucket.Delete(txHash[:]); err != nil {
// Remove the utxo entry if it is spent.
if entry.IsSpent() {
key := outpointKey(outpoint)
err := utxoBucket.Delete(*key)
recycleOutpointKey(key)
if err != nil {
return err
}
continue
}
// At this point the utxo entry is not fully spent, so store its
// serialization in the database.
err = utxoBucket.Put(txHash[:], serialized)
// Serialize and store the utxo entry.
serialized, err := serializeUtxoEntry(entry)
if err != nil {
return err
}
key := outpointKey(outpoint)
err = utxoBucket.Put(*key, serialized)
// NOTE: The key is intentionally not recycled here since the
// database interface contract prohibits modifications. It will
// be garbage collected normally when the database is done with
// it.
if err != nil {
return err
}
@ -1111,19 +1016,31 @@ func (b *BlockChain) createChainState() error {
return err
}
// Create the bucket that houses the spend journal data.
// Create the bucket that houses the spend journal data and
// store its version.
_, err = meta.CreateBucket(spendJournalBucketName)
if err != nil {
return err
}
err = dbPutVersion(dbTx, utxoSetVersionKeyName,
latestUtxoSetBucketVersion)
if err != nil {
return err
}
// Create the bucket that houses the utxo set. Note that the
// genesis block coinbase transaction is intentionally not
// inserted here since it is not spendable by consensus rules.
// Create the bucket that houses the utxo set and store its
// version. Note that the genesis block coinbase transaction is
// intentionally not inserted here since it is not spendable by
// consensus rules.
_, err = meta.CreateBucket(utxoSetBucketName)
if err != nil {
return err
}
err = dbPutVersion(dbTx, spendJournalVersionKeyName,
latestSpendJournalBucketVersion)
if err != nil {
return err
}
// Save the genesis block to the block index database.
err = dbStoreBlockNode(dbTx, node)

487
blockchain/chainio_test.go
View file

@ -11,7 +11,6 @@ import (
"reflect"
"testing"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/database"
"github.com/btcsuite/btcd/wire"
)
@ -38,19 +37,6 @@ func TestErrNotInMainChain(t *testing.T) {
}
}
// maybeDecompress decompresses the amount and public key script fields of the
// stxo and marks it decompressed if needed.
func (o *spentTxOut) maybeDecompress(version int32) {
// Nothing to do if it's not compressed.
if !o.compressed {
return
}
o.amount = int64(decompressTxOutAmount(uint64(o.amount)))
o.pkScript = decompressScript(o.pkScript, version)
o.compressed = false
}
// TestStxoSerialization ensures serializing and deserializing spent transaction
// output entries works as expected.
func TestStxoSerialization(t *testing.T) {
@ -59,7 +45,6 @@ func TestStxoSerialization(t *testing.T) {
tests := []struct {
name string
stxo spentTxOut
txVersion int32 // When the txout is not fully spent.
serialized []byte
}{
// From block 170 in main blockchain.
@ -70,9 +55,8 @@ func TestStxoSerialization(t *testing.T) {
pkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"),
isCoinBase: true,
height: 9,
version: 1,
},
serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"),
serialized: hexToBytes("1300320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"),
},
// Adapted from block 100025 in main blockchain.
{
@ -82,19 +66,16 @@ func TestStxoSerialization(t *testing.T) {
pkScript: hexToBytes("76a914b2fb57eadf61e106a100a7445a8c3f67898841ec88ac"),
isCoinBase: false,
height: 100024,
version: 1,
},
serialized: hexToBytes("8b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec"),
serialized: hexToBytes("8b99700086c64700b2fb57eadf61e106a100a7445a8c3f67898841ec"),
},
// Adapted from block 100025 in main blockchain.
{
name: "Does not spend last output",
name: "Does not spend last output, legacy format",
stxo: spentTxOut{
amount: 34405000000,
pkScript: hexToBytes("76a9146edbc6c4d31bae9f1ccc38538a114bf42de65e8688ac"),
version: 1,
},
txVersion: 1,
serialized: hexToBytes("0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e86"),
},
}
@ -130,14 +111,12 @@ func TestStxoSerialization(t *testing.T) {
// Ensure the serialized bytes are decoded back to the expected
// stxo.
var gotStxo spentTxOut
gotBytesRead, err := decodeSpentTxOut(test.serialized, &gotStxo,
test.txVersion)
gotBytesRead, err := decodeSpentTxOut(test.serialized, &gotStxo)
if err != nil {
t.Errorf("decodeSpentTxOut (%s): unexpected error: %v",
test.name, err)
continue
}
gotStxo.maybeDecompress(test.stxo.version)
if !reflect.DeepEqual(gotStxo, test.stxo) {
t.Errorf("decodeSpentTxOut (%s) mismatched entries - "+
"got %v, want %v", test.name, gotStxo, test.stxo)
@ -160,7 +139,6 @@ func TestStxoDecodeErrors(t *testing.T) {
tests := []struct {
name string
stxo spentTxOut
txVersion int32 // When the txout is not fully spent.
serialized []byte
bytesRead int // Expected number of bytes read.
errType error
@ -173,39 +151,30 @@ func TestStxoDecodeErrors(t *testing.T) {
bytesRead: 0,
},
{
name: "no data after header code w/o version",
name: "no data after header code w/o reserved",
stxo: spentTxOut{},
serialized: hexToBytes("00"),
errType: errDeserialize(""),
bytesRead: 1,
},
{
name: "no data after header code with version",
name: "no data after header code with reserved",
stxo: spentTxOut{},
serialized: hexToBytes("13"),
errType: errDeserialize(""),
bytesRead: 1,
},
{
name: "no data after version",
name: "no data after reserved",
stxo: spentTxOut{},
serialized: hexToBytes("1301"),
serialized: hexToBytes("1300"),
errType: errDeserialize(""),
bytesRead: 2,
},
{
name: "no serialized tx version and passed -1",
stxo: spentTxOut{},
txVersion: -1,
serialized: hexToBytes("003205"),
errType: AssertError(""),
bytesRead: 1,
},
{
name: "incomplete compressed txout",
stxo: spentTxOut{},
txVersion: 1,
serialized: hexToBytes("0032"),
serialized: hexToBytes("1332"),
errType: errDeserialize(""),
bytesRead: 2,
},
@ -214,7 +183,7 @@ func TestStxoDecodeErrors(t *testing.T) {
for _, test := range tests {
// Ensure the expected error type is returned.
gotBytesRead, err := decodeSpentTxOut(test.serialized,
&test.stxo, test.txVersion)
&test.stxo)
if reflect.TypeOf(err) != reflect.TypeOf(test.errType) {
t.Errorf("decodeSpentTxOut (%s): expected error type "+
"does not match - got %T, want %T", test.name,
@ -241,7 +210,6 @@ func TestSpendJournalSerialization(t *testing.T) {
name string
entry []spentTxOut
blockTxns []*wire.MsgTx
utxoView *UtxoViewpoint
serialized []byte
}{
// From block 2 in main blockchain.
@ -249,7 +217,6 @@ func TestSpendJournalSerialization(t *testing.T) {
name: "No spends",
entry: nil,
blockTxns: nil,
utxoView: NewUtxoViewpoint(),
serialized: nil,
},
// From block 170 in main blockchain.
@ -260,7 +227,6 @@ func TestSpendJournalSerialization(t *testing.T) {
pkScript: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"),
isCoinBase: true,
height: 9,
version: 1,
}},
blockTxns: []*wire.MsgTx{{ // Coinbase omitted.
Version: 1,
@ -281,22 +247,21 @@ func TestSpendJournalSerialization(t *testing.T) {
}},
LockTime: 0,
}},
utxoView: NewUtxoViewpoint(),
serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"),
serialized: hexToBytes("1300320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"),
},
// Adapted from block 100025 in main blockchain.
{
name: "Two txns when one spends last output, one doesn't",
entry: []spentTxOut{{
amount: 34405000000,
pkScript: hexToBytes("76a9146edbc6c4d31bae9f1ccc38538a114bf42de65e8688ac"),
version: 1,
amount: 34405000000,
pkScript: hexToBytes("76a9146edbc6c4d31bae9f1ccc38538a114bf42de65e8688ac"),
isCoinBase: false,
height: 100024,
}, {
amount: 13761000000,
pkScript: hexToBytes("76a914b2fb57eadf61e106a100a7445a8c3f67898841ec88ac"),
isCoinBase: false,
height: 100024,
version: 1,
}},
blockTxns: []*wire.MsgTx{{ // Coinbase omitted.
Version: 1,
@ -335,73 +300,7 @@ func TestSpendJournalSerialization(t *testing.T) {
}},
LockTime: 0,
}},
utxoView: &UtxoViewpoint{entries: map[chainhash.Hash]*UtxoEntry{
*newHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"): {
version: 1,
isCoinBase: false,
blockHeight: 100024,
sparseOutputs: map[uint32]*utxoOutput{
1: {
amount: 34405000000,
pkScript: hexToBytes("76a9142084541c3931677527a7eafe56fd90207c344eb088ac"),
},
},
},
}},
serialized: hexToBytes("8b99700186c64700b2fb57eadf61e106a100a7445a8c3f67898841ec0091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e86"),
},
// Hand crafted.
{
name: "One tx, two inputs from same tx, neither spend last output",
entry: []spentTxOut{{
amount: 165125632,
pkScript: hexToBytes("51"),
version: 1,
}, {
amount: 154370000,
pkScript: hexToBytes("51"),
version: 1,
}},
blockTxns: []*wire.MsgTx{{ // Coinbase omitted.
Version: 1,
TxIn: []*wire.TxIn{{
PreviousOutPoint: wire.OutPoint{
Hash: *newHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"),
Index: 1,
},
SignatureScript: hexToBytes(""),
Sequence: 0xffffffff,
}, {
PreviousOutPoint: wire.OutPoint{
Hash: *newHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"),
Index: 2,
},
SignatureScript: hexToBytes(""),
Sequence: 0xffffffff,
}},
TxOut: []*wire.TxOut{{
Value: 165125632,
PkScript: hexToBytes("51"),
}, {
Value: 154370000,
PkScript: hexToBytes("51"),
}},
LockTime: 0,
}},
utxoView: &UtxoViewpoint{entries: map[chainhash.Hash]*UtxoEntry{
*newHashFromStr("c0ed017828e59ad5ed3cf70ee7c6fb0f426433047462477dc7a5d470f987a537"): {
version: 1,
isCoinBase: false,
blockHeight: 100000,
sparseOutputs: map[uint32]*utxoOutput{
0: {
amount: 165712179,
pkScript: hexToBytes("51"),
},
},
},
}},
serialized: hexToBytes("0087bc3707510084c3d19a790751"),
serialized: hexToBytes("8b99700086c64700b2fb57eadf61e106a100a7445a8c3f67898841ec8b99700091f20f006edbc6c4d31bae9f1ccc38538a114bf42de65e86"),
},
}
@ -417,16 +316,12 @@ func TestSpendJournalSerialization(t *testing.T) {
// Deserialize to a spend journal entry.
gotEntry, err := deserializeSpendJournalEntry(test.serialized,
test.blockTxns, test.utxoView)
test.blockTxns)
if err != nil {
t.Errorf("deserializeSpendJournalEntry #%d (%s) "+
"unexpected error: %v", i, test.name, err)
continue
}
for stxoIdx := range gotEntry {
stxo := &gotEntry[stxoIdx]
stxo.maybeDecompress(test.entry[stxoIdx].version)
}
// Ensure that the deserialized spend journal entry has the
// correct properties.
@ -447,7 +342,6 @@ func TestSpendJournalErrors(t *testing.T) {
tests := []struct {
name string
blockTxns []*wire.MsgTx
utxoView *UtxoViewpoint
serialized []byte
errType error
}{
@ -466,7 +360,6 @@ func TestSpendJournalErrors(t *testing.T) {
}},
LockTime: 0,
}},
utxoView: NewUtxoViewpoint(),
serialized: hexToBytes(""),
errType: AssertError(""),
},
@ -484,7 +377,6 @@ func TestSpendJournalErrors(t *testing.T) {
}},
LockTime: 0,
}},
utxoView: NewUtxoViewpoint(),
serialized: hexToBytes("1301320511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a"),
errType: errDeserialize(""),
},
@ -494,7 +386,7 @@ func TestSpendJournalErrors(t *testing.T) {
// Ensure the expected error type is returned and the returned
// slice is nil.
stxos, err := deserializeSpendJournalEntry(test.serialized,
test.blockTxns, test.utxoView)
test.blockTxns)
if reflect.TypeOf(err) != reflect.TypeOf(test.errType) {
t.Errorf("deserializeSpendJournalEntry (%s): expected "+
"error type does not match - got %T, want %T",
@ -521,186 +413,52 @@ func TestUtxoSerialization(t *testing.T) {
serialized []byte
}{
// From tx in main blockchain:
// 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098
// 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098:0
{
name: "Only output 0, coinbase",
name: "height 1, coinbase",
entry: &UtxoEntry{
version: 1,
isCoinBase: true,
amount: 5000000000,
pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
blockHeight: 1,
sparseOutputs: map[uint32]*utxoOutput{
0: {
amount: 5000000000,
pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
},
},
packedFlags: tfCoinBase,
},
serialized: hexToBytes("010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"),
serialized: hexToBytes("03320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"),
},
// From tx in main blockchain:
// 8131ffb0a2c945ecaf9b9063e59558784f9c3a74741ce6ae2a18d0571dac15bb
// 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098:0
{
name: "Only output 1, not coinbase",
name: "height 1, coinbase, spent",
entry: &UtxoEntry{
version: 1,
isCoinBase: false,
blockHeight: 100001,
sparseOutputs: map[uint32]*utxoOutput{
1: {
amount: 1000000,
pkScript: hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
},
},
},
serialized: hexToBytes("01858c21040700ee8bd501094a7d5ca318da2506de35e1cb025ddc"),
},
// Adapted from tx in main blockchain:
// df3f3f442d9699857f7f49de4ff0b5d0f3448bec31cdc7b5bf6d25f2abd637d5
{
name: "Only output 2, coinbase",
entry: &UtxoEntry{
version: 1,
isCoinBase: true,
blockHeight: 99004,
sparseOutputs: map[uint32]*utxoOutput{
2: {
amount: 100937281,
pkScript: hexToBytes("76a914da33f77cee27c2a975ed5124d7e4f7f97513510188ac"),
},
},
},
serialized: hexToBytes("0185843c010182b095bf4100da33f77cee27c2a975ed5124d7e4f7f975135101"),
},
// Adapted from tx in main blockchain:
// 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f
{
name: "outputs 0 and 2 not coinbase",
entry: &UtxoEntry{
version: 1,
isCoinBase: false,
blockHeight: 113931,
sparseOutputs: map[uint32]*utxoOutput{
0: {
amount: 20000000,
pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"),
},
2: {
amount: 15000000,
pkScript: hexToBytes("76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac"),
},
},
},
serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"),
},
// Adapted from tx in main blockchain:
// 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f
{
name: "outputs 0 and 2, not coinbase, 1 marked spent",
entry: &UtxoEntry{
version: 1,
isCoinBase: false,
blockHeight: 113931,
sparseOutputs: map[uint32]*utxoOutput{
0: {
amount: 20000000,
pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"),
},
1: { // This won't be serialized.
spent: true,
amount: 1000000,
pkScript: hexToBytes("76a914e43031c3e46f20bf1ccee9553ce815de5a48467588ac"),
},
2: {
amount: 15000000,
pkScript: hexToBytes("76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac"),
},
},
},
serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"),
},
// Adapted from tx in main blockchain:
// 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f
{
name: "outputs 0 and 2, not coinbase, output 2 compressed",
entry: &UtxoEntry{
version: 1,
isCoinBase: false,
blockHeight: 113931,
sparseOutputs: map[uint32]*utxoOutput{
0: {
amount: 20000000,
pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"),
},
2: {
// Uncompressed Amount: 15000000
// Uncompressed PkScript: 76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac
compressed: true,
amount: 137,
pkScript: hexToBytes("00b8025be1b3efc63b0ad48e7f9f10e87544528d58"),
},
},
},
serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"),
},
// Adapted from tx in main blockchain:
// 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f
{
name: "outputs 0 and 2, not coinbase, output 2 compressed, packed indexes reversed",
entry: &UtxoEntry{
version: 1,
isCoinBase: false,
blockHeight: 113931,
sparseOutputs: map[uint32]*utxoOutput{
0: {
amount: 20000000,
pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"),
},
2: {
// Uncompressed Amount: 15000000
// Uncompressed PkScript: 76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac
compressed: true,
amount: 137,
pkScript: hexToBytes("00b8025be1b3efc63b0ad48e7f9f10e87544528d58"),
},
},
},
serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"),
},
// From tx in main blockchain:
// 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098
{
name: "Only output 0, coinbase, fully spent",
entry: &UtxoEntry{
version: 1,
isCoinBase: true,
amount: 5000000000,
pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
blockHeight: 1,
sparseOutputs: map[uint32]*utxoOutput{
0: {
spent: true,
amount: 5000000000,
pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
},
},
packedFlags: tfCoinBase | tfSpent,
},
serialized: nil,
},
// Adapted from tx in main blockchain:
// 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620
// From tx in main blockchain:
// 8131ffb0a2c945ecaf9b9063e59558784f9c3a74741ce6ae2a18d0571dac15bb:1
{
name: "Only output 22, not coinbase",
name: "height 100001, not coinbase",
entry: &UtxoEntry{
version: 1,
isCoinBase: false,
blockHeight: 338156,
sparseOutputs: map[uint32]*utxoOutput{
22: {
spent: false,
amount: 366875659,
pkScript: hexToBytes("a9141dd46a006572d820e448e12d2bbb38640bc718e687"),
},
},
amount: 1000000,
pkScript: hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
blockHeight: 100001,
packedFlags: 0,
},
serialized: hexToBytes("0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6"),
serialized: hexToBytes("8b99420700ee8bd501094a7d5ca318da2506de35e1cb025ddc"),
},
// From tx in main blockchain:
// 8131ffb0a2c945ecaf9b9063e59558784f9c3a74741ce6ae2a18d0571dac15bb:1
{
name: "height 100001, not coinbase, spent",
entry: &UtxoEntry{
amount: 1000000,
pkScript: hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
blockHeight: 100001,
packedFlags: tfSpent,
},
serialized: nil,
},
}
@ -719,9 +477,9 @@ func TestUtxoSerialization(t *testing.T) {
continue
}
// Don't try to deserialize if the test entry was fully spent
// since it will have a nil serialization.
if test.entry.IsFullySpent() {
// Don't try to deserialize if the test entry was spent since it
// will have a nil serialization.
if test.entry.IsSpent() {
continue
}
@ -733,12 +491,33 @@ func TestUtxoSerialization(t *testing.T) {
continue
}
// Ensure that the deserialized utxo entry has the same
// properties for the containing transaction and block height.
if utxoEntry.Version() != test.entry.Version() {
// The deserialized entry must not be marked spent since unspent
// entries are not serialized.
if utxoEntry.IsSpent() {
t.Errorf("deserializeUtxoEntry #%d (%s) output should "+
"not be marked spent", i, test.name)
continue
}
// Ensure the deserialized entry has the same properties as the
// ones in the test entry.
if utxoEntry.Amount() != test.entry.Amount() {
t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+
"version: got %d, want %d", i, test.name,
utxoEntry.Version(), test.entry.Version())
"amounts: got %d, want %d", i, test.name,
utxoEntry.Amount(), test.entry.Amount())
continue
}
if !bytes.Equal(utxoEntry.PkScript(), test.entry.PkScript()) {
t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+
"scripts: got %x, want %x", i, test.name,
utxoEntry.PkScript(), test.entry.PkScript())
continue
}
if utxoEntry.BlockHeight() != test.entry.BlockHeight() {
t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+
"block height: got %d, want %d", i, test.name,
utxoEntry.BlockHeight(), test.entry.BlockHeight())
continue
}
if utxoEntry.IsCoinBase() != test.entry.IsCoinBase() {
@ -747,71 +526,6 @@ func TestUtxoSerialization(t *testing.T) {
utxoEntry.IsCoinBase(), test.entry.IsCoinBase())
continue
}
if utxoEntry.BlockHeight() != test.entry.BlockHeight() {
t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+
"block height: got %d, want %d", i, test.name,
utxoEntry.BlockHeight(),
test.entry.BlockHeight())
continue
}
if utxoEntry.IsFullySpent() != test.entry.IsFullySpent() {
t.Errorf("deserializeUtxoEntry #%d (%s) mismatched "+
"fully spent: got %v, want %v", i, test.name,
utxoEntry.IsFullySpent(),
test.entry.IsFullySpent())
continue
}
// Ensure all of the outputs in the test entry match the
// spentness of the output in the deserialized entry and the
// deserialized entry does not contain any additional utxos.
var numUnspent int
for outputIndex := range test.entry.sparseOutputs {
gotSpent := utxoEntry.IsOutputSpent(outputIndex)
wantSpent := test.entry.IsOutputSpent(outputIndex)
if !wantSpent {
numUnspent++
}
if gotSpent != wantSpent {
t.Errorf("deserializeUtxoEntry #%d (%s) output "+
"#%d: mismatched spent: got %v, want "+
"%v", i, test.name, outputIndex,
gotSpent, wantSpent)
continue
}
}
if len(utxoEntry.sparseOutputs) != numUnspent {
t.Errorf("deserializeUtxoEntry #%d (%s): mismatched "+
"number of unspent outputs: got %d, want %d", i,
test.name, len(utxoEntry.sparseOutputs),
numUnspent)
continue
}
// Ensure all of the amounts and scripts of the utxos in the
// deserialized entry match the ones in the test entry.
for outputIndex := range utxoEntry.sparseOutputs {
gotAmount := utxoEntry.AmountByIndex(outputIndex)
wantAmount := test.entry.AmountByIndex(outputIndex)
if gotAmount != wantAmount {
t.Errorf("deserializeUtxoEntry #%d (%s) "+
"output #%d: mismatched amounts: got "+
"%d, want %d", i, test.name,
outputIndex, gotAmount, wantAmount)
continue
}
gotPkScript := utxoEntry.PkScriptByIndex(outputIndex)
wantPkScript := test.entry.PkScriptByIndex(outputIndex)
if !bytes.Equal(gotPkScript, wantPkScript) {
t.Errorf("deserializeUtxoEntry #%d (%s) "+
"output #%d mismatched scripts: got "+
"%x, want %x", i, test.name,
outputIndex, gotPkScript, wantPkScript)
continue
}
}
}
}
@ -821,23 +535,21 @@ func TestUtxoEntryHeaderCodeErrors(t *testing.T) {
t.Parallel()
tests := []struct {
name string
entry *UtxoEntry
code uint64
bytesRead int // Expected number of bytes read.
errType error
name string
entry *UtxoEntry
code uint64
errType error
}{
{
name: "Force assertion due to fully spent tx",
entry: &UtxoEntry{},
errType: AssertError(""),
bytesRead: 0,
name: "Force assertion due to spent output",
entry: &UtxoEntry{packedFlags: tfSpent},
errType: AssertError(""),
},
}
for _, test := range tests {
// Ensure the expected error type is returned and the code is 0.
code, gotBytesRead, err := utxoEntryHeaderCode(test.entry, 0)
code, err := utxoEntryHeaderCode(test.entry)
if reflect.TypeOf(err) != reflect.TypeOf(test.errType) {
t.Errorf("utxoEntryHeaderCode (%s): expected error "+
"type does not match - got %T, want %T",
@ -849,14 +561,6 @@ func TestUtxoEntryHeaderCodeErrors(t *testing.T) {
"on error - got %d, want 0", test.name, code)
continue
}
// Ensure the expected number of bytes read is returned.
if gotBytesRead != test.bytesRead {
t.Errorf("utxoEntryHeaderCode (%s): unexpected number "+
"of bytes read - got %d, want %d", test.name,
gotBytesRead, test.bytesRead)
continue
}
}
}
@ -870,29 +574,14 @@ func TestUtxoEntryDeserializeErrors(t *testing.T) {
serialized []byte
errType error
}{
{
name: "no data after version",
serialized: hexToBytes("01"),
errType: errDeserialize(""),
},
{
name: "no data after block height",
serialized: hexToBytes("0101"),
errType: errDeserialize(""),
},
{
name: "no data after header code",
serialized: hexToBytes("010102"),
errType: errDeserialize(""),
},
{
name: "not enough bytes for unspentness bitmap",
serialized: hexToBytes("01017800"),
serialized: hexToBytes("02"),
errType: errDeserialize(""),
},
{
name: "incomplete compressed txout",
serialized: hexToBytes("01010232"),
serialized: hexToBytes("0232"),
errType: errDeserialize(""),
},
}

101
blockchain/common_test.go
View file

@ -190,10 +190,10 @@ func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), er
// loadUtxoView returns a utxo view loaded from a file.
func loadUtxoView(filename string) (*UtxoViewpoint, error) {
// The utxostore file format is:
// <tx hash><serialized utxo len><serialized utxo>
// <tx hash><output index><serialized utxo len><serialized utxo>
//
// The serialized utxo len is a little endian uint32 and the serialized
// utxo uses the format described in chainio.go.
// The output index and serialized utxo len are little endian uint32s
// and the serialized utxo uses the format described in chainio.go.
filename = filepath.Join("testdata", filename)
fi, err := os.Open(filename)
@ -223,7 +223,14 @@ func loadUtxoView(filename string) (*UtxoViewpoint, error) {
return nil, err
}
// Num of serialize utxo entry bytes.
// Output index of the utxo entry.
var index uint32
err = binary.Read(r, binary.LittleEndian, &index)
if err != nil {
return nil, err
}
// Num of serialized utxo entry bytes.
var numBytes uint32
err = binary.Read(r, binary.LittleEndian, &numBytes)
if err != nil {
@ -238,16 +245,98 @@ func loadUtxoView(filename string) (*UtxoViewpoint, error) {
}
// Deserialize it and add it to the view.
utxoEntry, err := deserializeUtxoEntry(serialized)
entry, err := deserializeUtxoEntry(serialized)
if err != nil {
return nil, err
}
view.Entries()[hash] = utxoEntry
view.Entries()[wire.OutPoint{Hash: hash, Index: index}] = entry
}
return view, nil
}
// convertUtxoStore reads a utxostore from the legacy format and writes it back
// out using the latest format. It is only useful for converting utxostore data
// used in the tests, which has already been done. However, the code is left
// available for future reference.
func convertUtxoStore(r io.Reader, w io.Writer) error {
// The old utxostore file format was:
// <tx hash><serialized utxo len><serialized utxo>
//
// The serialized utxo len was a little endian uint32 and the serialized
// utxo uses the format described in upgrade.go.
littleEndian := binary.LittleEndian
for {
// Hash of the utxo entry.
var hash chainhash.Hash
_, err := io.ReadAtLeast(r, hash[:], len(hash[:]))
if err != nil {
// Expected EOF at the right offset.
if err == io.EOF {
break
}
return err
}
// Num of serialized utxo entry bytes.
var numBytes uint32
err = binary.Read(r, littleEndian, &numBytes)
if err != nil {
return err
}
// Serialized utxo entry.
serialized := make([]byte, numBytes)
_, err = io.ReadAtLeast(r, serialized, int(numBytes))
if err != nil {
return err
}
// Deserialize the entry.
entries, err := deserializeUtxoEntryV0(serialized)
if err != nil {
return err
}
// Loop through all of the utxos and write them out in the new
// format.
for outputIdx, entry := range entries {
// Reserialize the entries using the new format.
serialized, err := serializeUtxoEntry(entry)
if err != nil {
return err
}
// Write the hash of the utxo entry.
_, err = w.Write(hash[:])
if err != nil {
return err
}
// Write the output index of the utxo entry.
err = binary.Write(w, littleEndian, outputIdx)
if err != nil {
return err
}
// Write num of serialized utxo entry bytes.
err = binary.Write(w, littleEndian, uint32(len(serialized)))
if err != nil {
return err
}
// Write the serialized utxo.
_, err = w.Write(serialized)
if err != nil {
return err
}
}
}
return nil
}
// TstSetCoinbaseMaturity makes the ability to set the coinbase maturity
// available when running tests.
func (b *BlockChain) TstSetCoinbaseMaturity(maturity uint16) {

61
blockchain/compress.go
View file

@ -241,7 +241,7 @@ func isPubKey(script []byte) (bool, []byte) {
// compressedScriptSize returns the number of bytes the passed script would take
// when encoded with the domain specific compression algorithm described above.
func compressedScriptSize(pkScript []byte, version int32) int {
func compressedScriptSize(pkScript []byte) int {
// Pay-to-pubkey-hash script.
if valid, _ := isPubKeyHash(pkScript); valid {
return 21
@ -268,7 +268,7 @@ func compressedScriptSize(pkScript []byte, version int32) int {
// script, possibly followed by other data, and returns the number of bytes it
// occupies taking into account the special encoding of the script size by the
// domain specific compression algorithm described above.
func decodeCompressedScriptSize(serialized []byte, version int32) int {
func decodeCompressedScriptSize(serialized []byte) int {
scriptSize, bytesRead := deserializeVLQ(serialized)
if bytesRead == 0 {
return 0
@ -296,7 +296,7 @@ func decodeCompressedScriptSize(serialized []byte, version int32) int {
// target byte slice. The target byte slice must be at least large enough to
// handle the number of bytes returned by the compressedScriptSize function or
// it will panic.
func putCompressedScript(target, pkScript []byte, version int32) int {
func putCompressedScript(target, pkScript []byte) int {
// Pay-to-pubkey-hash script.
if valid, hash := isPubKeyHash(pkScript); valid {
target[0] = cstPayToPubKeyHash
@ -344,7 +344,7 @@ func putCompressedScript(target, pkScript []byte, version int32) int {
// NOTE: The script parameter must already have been proven to be long enough
// to contain the number of bytes returned by decodeCompressedScriptSize or it
// will panic. This is acceptable since it is only an internal function.
func decompressScript(compressedPkScript []byte, version int32) []byte {
func decompressScript(compressedPkScript []byte) []byte {
// In practice this function will not be called with a zero-length or
// nil script since the nil script encoding includes the length, however
// the code below assumes the length exists, so just return nil now if
@ -542,43 +542,27 @@ func decompressTxOutAmount(amount uint64) uint64 {
// -----------------------------------------------------------------------------
// compressedTxOutSize returns the number of bytes the passed transaction output
// fields would take when encoded with the format described above. The
// preCompressed flag indicates the provided amount and script are already
// compressed. This is useful since loaded utxo entries are not decompressed
// until the output is accessed.
func compressedTxOutSize(amount uint64, pkScript []byte, version int32, preCompressed bool) int {
if preCompressed {
return serializeSizeVLQ(amount) + len(pkScript)
}
// fields would take when encoded with the format described above.
func compressedTxOutSize(amount uint64, pkScript []byte) int {
return serializeSizeVLQ(compressTxOutAmount(amount)) +
compressedScriptSize(pkScript, version)
compressedScriptSize(pkScript)
}
// putCompressedTxOut potentially compresses the passed amount and script
// according to their domain specific compression algorithms and encodes them
// directly into the passed target byte slice with the format described above.
// The preCompressed flag indicates the provided amount and script are already
// compressed in which case the values are not modified. This is useful since
// loaded utxo entries are not decompressed until the output is accessed. The
// target byte slice must be at least large enough to handle the number of bytes
// returned by the compressedTxOutSize function or it will panic.
func putCompressedTxOut(target []byte, amount uint64, pkScript []byte, version int32, preCompressed bool) int {
if preCompressed {
offset := putVLQ(target, amount)
copy(target[offset:], pkScript)
return offset + len(pkScript)
}
// putCompressedTxOut compresses the passed amount and script according to their
// domain specific compression algorithms and encodes them directly into the
// passed target byte slice with the format described above. The target byte
// slice must be at least large enough to handle the number of bytes returned by
// the compressedTxOutSize function or it will panic.
func putCompressedTxOut(target []byte, amount uint64, pkScript []byte) int {
offset := putVLQ(target, compressTxOutAmount(amount))
offset += putCompressedScript(target[offset:], pkScript, version)
offset += putCompressedScript(target[offset:], pkScript)
return offset
}
// decodeCompressedTxOut decodes the passed compressed txout, possibly followed
// by other data, into its compressed amount and compressed script and returns
// them along with the number of bytes they occupied.
func decodeCompressedTxOut(serialized []byte, version int32) (uint64, []byte, int, error) {
// by other data, into its uncompressed amount and script and returns them along
// with the number of bytes they occupied prior to decompression.
func decodeCompressedTxOut(serialized []byte) (uint64, []byte, int, error) {
// Deserialize the compressed amount and ensure there are bytes
// remaining for the compressed script.
compressedAmount, bytesRead := deserializeVLQ(serialized)
@ -589,15 +573,14 @@ func decodeCompressedTxOut(serialized []byte, version int32) (uint64, []byte, in
// Decode the compressed script size and ensure there are enough bytes
// left in the slice for it.
scriptSize := decodeCompressedScriptSize(serialized[bytesRead:], version)
scriptSize := decodeCompressedScriptSize(serialized[bytesRead:])
if len(serialized[bytesRead:]) < scriptSize {
return 0, nil, bytesRead, errDeserialize("unexpected end of " +
"data after script size")
}
// Make a copy of the compressed script so the original serialized data
// can be released as soon as possible.
compressedScript := make([]byte, scriptSize)
copy(compressedScript, serialized[bytesRead:bytesRead+scriptSize])
return compressedAmount, compressedScript, bytesRead + scriptSize, nil
// Decompress and return the amount and script.
amount := decompressTxOutAmount(compressedAmount)
script := decompressScript(serialized[bytesRead : bytesRead+scriptSize])
return amount, script, bytesRead + scriptSize, nil
}

110
blockchain/compress_test.go
View file

@ -109,79 +109,66 @@ func TestScriptCompression(t *testing.T) {
tests := []struct {
name string
version int32
uncompressed []byte
compressed []byte
}{
{
name: "nil",
version: 1,
uncompressed: nil,
compressed: hexToBytes("06"),
},
{
name: "pay-to-pubkey-hash 1",
version: 1,
uncompressed: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
compressed: hexToBytes("001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
},
{
name: "pay-to-pubkey-hash 2",
version: 1,
uncompressed: hexToBytes("76a914e34cce70c86373273efcc54ce7d2a491bb4a0e8488ac"),
compressed: hexToBytes("00e34cce70c86373273efcc54ce7d2a491bb4a0e84"),
},
{
name: "pay-to-script-hash 1",
version: 1,
uncompressed: hexToBytes("a914da1745e9b549bd0bfa1a569971c77eba30cd5a4b87"),
compressed: hexToBytes("01da1745e9b549bd0bfa1a569971c77eba30cd5a4b"),
},
{
name: "pay-to-script-hash 2",
version: 1,
uncompressed: hexToBytes("a914f815b036d9bbbce5e9f2a00abd1bf3dc91e9551087"),
compressed: hexToBytes("01f815b036d9bbbce5e9f2a00abd1bf3dc91e95510"),
},
{
name: "pay-to-pubkey compressed 0x02",
version: 1,
uncompressed: hexToBytes("2102192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4ac"),
compressed: hexToBytes("02192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
},
{
name: "pay-to-pubkey compressed 0x03",
version: 1,
uncompressed: hexToBytes("2103b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e65ac"),
compressed: hexToBytes("03b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e65"),
},
{
name: "pay-to-pubkey uncompressed 0x04 even",
version: 1,
uncompressed: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
compressed: hexToBytes("04192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
},
{
name: "pay-to-pubkey uncompressed 0x04 odd",
version: 1,
uncompressed: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"),
compressed: hexToBytes("0511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"),
},
{
name: "pay-to-pubkey invalid pubkey",
version: 1,
uncompressed: hexToBytes("3302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
compressed: hexToBytes("293302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
},
{
name: "null data",
version: 1,
uncompressed: hexToBytes("6a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
compressed: hexToBytes("286a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
},
{
name: "requires 2 size bytes - data push 200 bytes",
version: 1,
uncompressed: append(hexToBytes("4cc8"), bytes.Repeat([]byte{0x00}, 200)...),
// [0x80, 0x50] = 208 as a variable length quantity
// [0x4c, 0xc8] = OP_PUSHDATA1 200
@ -192,7 +179,7 @@ func TestScriptCompression(t *testing.T) {
for _, test := range tests {
// Ensure the function to calculate the serialized size without
// actually serializing the value is calculated properly.
gotSize := compressedScriptSize(test.uncompressed, test.version)
gotSize := compressedScriptSize(test.uncompressed)
if gotSize != len(test.compressed) {
t.Errorf("compressedScriptSize (%s): did not get "+
"expected size - got %d, want %d", test.name,
@ -203,7 +190,7 @@ func TestScriptCompression(t *testing.T) {
// Ensure the script compresses to the expected bytes.
gotCompressed := make([]byte, gotSize)
gotBytesWritten := putCompressedScript(gotCompressed,
test.uncompressed, test.version)
test.uncompressed)
if !bytes.Equal(gotCompressed, test.compressed) {
t.Errorf("putCompressedScript (%s): did not get "+
"expected bytes - got %x, want %x", test.name,
@ -220,8 +207,7 @@ func TestScriptCompression(t *testing.T) {
// Ensure the compressed script size is properly decoded from
// the compressed script.
gotDecodedSize := decodeCompressedScriptSize(test.compressed,
test.version)
gotDecodedSize := decodeCompressedScriptSize(test.compressed)
if gotDecodedSize != len(test.compressed) {
t.Errorf("decodeCompressedScriptSize (%s): did not get "+
"expected size - got %d, want %d", test.name,
@ -230,7 +216,7 @@ func TestScriptCompression(t *testing.T) {
}
// Ensure the script decompresses to the expected bytes.
gotDecompressed := decompressScript(test.compressed, test.version)
gotDecompressed := decompressScript(test.compressed)
if !bytes.Equal(gotDecompressed, test.uncompressed) {
t.Errorf("decompressScript (%s): did not get expected "+
"bytes - got %x, want %x", test.name,
@ -246,13 +232,13 @@ func TestScriptCompressionErrors(t *testing.T) {
t.Parallel()
// A nil script must result in a decoded size of 0.
if gotSize := decodeCompressedScriptSize(nil, 1); gotSize != 0 {
if gotSize := decodeCompressedScriptSize(nil); gotSize != 0 {
t.Fatalf("decodeCompressedScriptSize with nil script did not "+
"return 0 - got %d", gotSize)
}
// A nil script must result in a nil decompressed script.
if gotScript := decompressScript(nil, 1); gotScript != nil {
if gotScript := decompressScript(nil); gotScript != nil {
t.Fatalf("decompressScript with nil script did not return nil "+
"decompressed script - got %x", gotScript)
}
@ -261,7 +247,7 @@ func TestScriptCompressionErrors(t *testing.T) {
// in an invalid pubkey must result in a nil decompressed script.
compressedScript := hexToBytes("04012d74d0cb94344c9569c2e77901573d8d" +
"7903c3ebec3a957724895dca52c6b4")
if gotScript := decompressScript(compressedScript, 1); gotScript != nil {
if gotScript := decompressScript(compressedScript); gotScript != nil {
t.Fatalf("decompressScript with compressed pay-to-"+
"uncompressed-pubkey that is invalid did not return "+
"nil decompressed script - got %x", gotScript)
@ -352,48 +338,35 @@ func TestCompressedTxOut(t *testing.T) {
t.Parallel()
tests := []struct {
name string
amount uint64
compAmount uint64
pkScript []byte
compPkScript []byte
version int32
compressed []byte
name string
amount uint64
pkScript []byte
compressed []byte
}{
{
name: "nulldata with 0 BTC",
amount: 0,
compAmount: 0,
pkScript: hexToBytes("6a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
compPkScript: hexToBytes("286a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
version: 1,
compressed: hexToBytes("00286a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
name: "nulldata with 0 BTC",
amount: 0,
pkScript: hexToBytes("6a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
compressed: hexToBytes("00286a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
},
{
name: "pay-to-pubkey-hash dust",
amount: 546,
compAmount: 4911,
pkScript: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
compPkScript: hexToBytes("001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
version: 1,
compressed: hexToBytes("a52f001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
name: "pay-to-pubkey-hash dust",
amount: 546,
pkScript: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
compressed: hexToBytes("a52f001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
},
{
name: "pay-to-pubkey uncompressed 1 BTC",
amount: 100000000,
compAmount: 9,
pkScript: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
compPkScript: hexToBytes("04192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
version: 1,
compressed: hexToBytes("0904192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
name: "pay-to-pubkey uncompressed 1 BTC",
amount: 100000000,
pkScript: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
compressed: hexToBytes("0904192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
},
}
for _, test := range tests {
// Ensure the function to calculate the serialized size without
// actually serializing the txout is calculated properly.
gotSize := compressedTxOutSize(test.amount, test.pkScript,
test.version, false)
gotSize := compressedTxOutSize(test.amount, test.pkScript)
if gotSize != len(test.compressed) {
t.Errorf("compressedTxOutSize (%s): did not get "+
"expected size - got %d, want %d", test.name,
@ -404,7 +377,7 @@ func TestCompressedTxOut(t *testing.T) {
// Ensure the txout compresses to the expected value.
gotCompressed := make([]byte, gotSize)
gotBytesWritten := putCompressedTxOut(gotCompressed,
test.amount, test.pkScript, test.version, false)
test.amount, test.pkScript)
if !bytes.Equal(gotCompressed, test.compressed) {
t.Errorf("compressTxOut (%s): did not get expected "+
"bytes - got %x, want %x", test.name,
@ -420,24 +393,24 @@ func TestCompressedTxOut(t *testing.T) {
}
// Ensure the serialized bytes are decoded back to the expected
// compressed values.
// uncompressed values.
gotAmount, gotScript, gotBytesRead, err := decodeCompressedTxOut(
test.compressed, test.version)
test.compressed)
if err != nil {
t.Errorf("decodeCompressedTxOut (%s): unexpected "+
"error: %v", test.name, err)
continue
}
if gotAmount != test.compAmount {
if gotAmount != test.amount {
t.Errorf("decodeCompressedTxOut (%s): did not get "+
"expected amount - got %d, want %d",
test.name, gotAmount, test.compAmount)
test.name, gotAmount, test.amount)
continue
}
if !bytes.Equal(gotScript, test.compPkScript) {
if !bytes.Equal(gotScript, test.pkScript) {
t.Errorf("decodeCompressedTxOut (%s): did not get "+
"expected script - got %x, want %x",
test.name, gotScript, test.compPkScript)
test.name, gotScript, test.pkScript)
continue
}
if gotBytesRead != len(test.compressed) {
@ -446,23 +419,6 @@ func TestCompressedTxOut(t *testing.T) {
test.name, gotBytesRead, len(test.compressed))
continue
}
// Ensure the compressed values decompress to the expected
// txout.
gotAmount = decompressTxOutAmount(gotAmount)
if gotAmount != test.amount {
t.Errorf("decompressTxOut (%s): did not get expected "+
"value - got %d, want %d", test.name, gotAmount,
test.amount)
continue
}
gotScript = decompressScript(gotScript, test.version)
if !bytes.Equal(gotScript, test.pkScript) {
t.Errorf("decompressTxOut (%s): did not get expected "+
"script - got %x, want %x", test.name,
gotScript, test.pkScript)
continue
}
}
}
@ -473,7 +429,7 @@ func TestTxOutCompressionErrors(t *testing.T) {
// A compressed txout with missing compressed script must error.
compressedTxOut := hexToBytes("00")
_, _, _, err := decodeCompressedTxOut(compressedTxOut, 1)
_, _, _, err := decodeCompressedTxOut(compressedTxOut)
if !isDeserializeErr(err) {
t.Fatalf("decodeCompressedTxOut with missing compressed script "+
"did not return expected error type - got %T, want "+
@ -482,7 +438,7 @@ func TestTxOutCompressionErrors(t *testing.T) {
// A compressed txout with short compressed script must error.
compressedTxOut = hexToBytes("0010")
_, _, _, err = decodeCompressedTxOut(compressedTxOut, 1)
_, _, _, err = decodeCompressedTxOut(compressedTxOut)
if !isDeserializeErr(err) {
t.Fatalf("decodeCompressedTxOut with short compressed script "+
"did not return expected error type - got %T, want "+

11
blockchain/indexers/addrindex.go
View file

@ -703,14 +703,12 @@ func (idx *AddrIndex) indexBlock(data writeIndexData, block *btcutil.Block, view
// The view should always have the input since
// the index contract requires it, however, be
// safe and simply ignore any missing entries.
origin := &txIn.PreviousOutPoint
entry := view.LookupEntry(&origin.Hash)
entry := view.LookupEntry(txIn.PreviousOutPoint)
if entry == nil {
continue
}
pkScript := entry.PkScriptByIndex(origin.Index)
idx.indexPkScript(data, pkScript, txIdx)
idx.indexPkScript(data, entry.PkScript(), txIdx)
}
}
@ -872,15 +870,14 @@ func (idx *AddrIndex) AddUnconfirmedTx(tx *btcutil.Tx, utxoView *blockchain.Utxo
// transaction has already been validated and thus all inputs are
// already known to exist.
for _, txIn := range tx.MsgTx().TxIn {
entry := utxoView.LookupEntry(&txIn.PreviousOutPoint.Hash)
entry := utxoView.LookupEntry(txIn.PreviousOutPoint)
if entry == nil {
// Ignore missing entries. This should never happen
// in practice since the function comments specifically
// call out all inputs must be available.
continue
}
pkScript := entry.PkScriptByIndex(txIn.PreviousOutPoint.Index)
idx.indexUnconfirmedAddresses(pkScript, tx)
idx.indexUnconfirmedAddresses(entry.PkScript(), tx)
}
// Index addresses of all created outputs.

42
blockchain/scriptval.go
View file

@ -55,31 +55,16 @@ out:
for {
select {
case txVI := <-v.validateChan:
// Ensure the referenced input transaction is available.
// Ensure the referenced input utxo is available.
txIn := txVI.txIn
originTxHash := &txIn.PreviousOutPoint.Hash
originTxIndex := txIn.PreviousOutPoint.Index
txEntry := v.utxoView.LookupEntry(originTxHash)
if txEntry == nil {
str := fmt.Sprintf("unable to find input "+
"transaction %v referenced from "+
"transaction %v", originTxHash,
txVI.tx.Hash())
err := ruleError(ErrMissingTxOut, str)
v.sendResult(err)
break out
}
// Ensure the referenced input transaction public key
// script is available.
pkScript := txEntry.PkScriptByIndex(originTxIndex)
if pkScript == nil {
utxo := v.utxoView.LookupEntry(txIn.PreviousOutPoint)
if utxo == nil {
str := fmt.Sprintf("unable to find unspent "+
"output %v script referenced from "+
"output %v referenced from "+
"transaction %s:%d",
txIn.PreviousOutPoint, txVI.tx.Hash(),
txVI.txInIndex)
err := ruleError(ErrBadTxInput, str)
err := ruleError(ErrMissingTxOut, str)
v.sendResult(err)
break out
}
@ -87,18 +72,19 @@ out:
// Create a new script engine for the script pair.
sigScript := txIn.SignatureScript
witness := txIn.Witness
inputAmount := txEntry.AmountByIndex(originTxIndex)
pkScript := utxo.PkScript()
inputAmount := utxo.Amount()
vm, err := txscript.NewEngine(pkScript, txVI.tx.MsgTx(),
txVI.txInIndex, v.flags, v.sigCache, txVI.sigHashes,
inputAmount)
if err != nil {
str := fmt.Sprintf("failed to parse input "+
"%s:%d which references output %s:%d - "+
"%s:%d which references output %v - "+
"%v (input witness %x, input script "+
"bytes %x, prev output script bytes %x)",
txVI.tx.Hash(), txVI.txInIndex, originTxHash,
originTxIndex, err, witness, sigScript,
pkScript)
txVI.tx.Hash(), txVI.txInIndex,
txIn.PreviousOutPoint, err, witness,
sigScript, pkScript)
err := ruleError(ErrScriptMalformed, str)
v.sendResult(err)
break out
@ -107,12 +93,12 @@ out:
// Execute the script pair.
if err := vm.Execute(); err != nil {
str := fmt.Sprintf("failed to validate input "+
"%s:%d which references output %s:%d - "+
"%s:%d which references output %v - "+
"%v (input witness %x, input script "+
"bytes %x, prev output script bytes %x)",
txVI.tx.Hash(), txVI.txInIndex,
originTxHash, originTxIndex, err,
witness, sigScript, pkScript)
txIn.PreviousOutPoint, err, witness,
sigScript, pkScript)
err := ruleError(ErrScriptValidation, str)
v.sendResult(err)
break out

BIN
blockchain/testdata/277647.utxostore.bz2 vendored
View file

Binary file not shown.

398
blockchain/upgrade.go
View file

@ -7,7 +7,9 @@ package blockchain
import (
"bytes"
"container/list"
"errors"
"fmt"
"time"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/database"
@ -23,6 +25,23 @@ const (
blockHdrOffset = 12
)
// errInterruptRequested indicates that an operation was cancelled due
// to a user-requested interrupt.
var errInterruptRequested = errors.New("interrupt requested")
// interruptRequested returns true when the provided channel has been closed.
// This simplifies early shutdown slightly since the caller can just use an if
// statement instead of a select.
func interruptRequested(interrupted <-chan struct{}) bool {
select {
case <-interrupted:
return true
default:
}
return false
}
// blockChainContext represents a particular block's placement in the block
// chain. This is used by the block index migration to track block metadata that
// will be written to disk.
@ -204,3 +223,382 @@ func determineMainChainBlocks(blocksMap map[chainhash.Hash]*blockChainContext, t
blocksMap[*nextHash].mainChain = true
}
}
// deserializeUtxoEntryV0 decodes a utxo entry from the passed serialized byte
// slice according to the legacy version 0 format into a map of utxos keyed by
// the output index within the transaction. The map is necessary because the
// previous format encoded all unspent outputs for a transaction using a single
// entry, whereas the new format encodes each unspent output individually.
//
// The legacy format is as follows:
//
// <version><height><header code><unspentness bitmap>[<compressed txouts>,...]
//
// Field Type Size
// version VLQ variable
// block height VLQ variable
// header code VLQ variable
// unspentness bitmap []byte variable
// compressed txouts
// compressed amount VLQ variable
// compressed script []byte variable
//
// The serialized header code format is:
// bit 0 - containing transaction is a coinbase
// bit 1 - output zero is unspent
// bit 2 - output one is unspent
// bits 3-x - number of bytes in unspentness bitmap. When both bits 1 and 2
// are unset, it encodes N-1 since there must be at least one unspent
// output.
//
// The rationale for the header code scheme is as follows:
// - Transactions which only pay to a single output and a change output are
// extremely common, thus an extra byte for the unspentness bitmap can be
// avoided for them by encoding those two outputs in the low order bits.
// - Given it is encoded as a VLQ which can encode values up to 127 with a
// single byte, that leaves 4 bits to represent the number of bytes in the
// unspentness bitmap while still only consuming a single byte for the
// header code. In other words, an unspentness bitmap with up to 120
// transaction outputs can be encoded with a single-byte header code.
// This covers the vast majority of transactions.
// - Encoding N-1 bytes when both bits 1 and 2 are unset allows an additional
// 8 outpoints to be encoded before causing the header code to require an
// additional byte.
//
// Example 1:
// From tx in main blockchain:
// Blk 1, 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098
//
// 010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52
// <><><><------------------------------------------------------------------>
// | | \--------\ |
// | height | compressed txout 0
// version header code
//
// - version: 1
// - height: 1
// - header code: 0x03 (coinbase, output zero unspent, 0 bytes of unspentness)
// - unspentness: Nothing since it is zero bytes
// - compressed txout 0:
// - 0x32: VLQ-encoded compressed amount for 5000000000 (50 BTC)
// - 0x04: special script type pay-to-pubkey
// - 0x96...52: x-coordinate of the pubkey
//
// Example 2:
// From tx in main blockchain:
// Blk 113931, 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f
//
// 0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58
// <><----><><><------------------------------------------><-------------------------------------------->
// | | | \-------------------\ | |
// version | \--------\ unspentness | compressed txout 2
// height header code compressed txout 0
//
// - version: 1
// - height: 113931
// - header code: 0x0a (output zero unspent, 1 byte in unspentness bitmap)
// - unspentness: [0x01] (bit 0 is set, so output 0+2 = 2 is unspent)
// NOTE: It's +2 since the first two outputs are encoded in the header code
// - compressed txout 0:
// - 0x12: VLQ-encoded compressed amount for 20000000 (0.2 BTC)
// - 0x00: special script type pay-to-pubkey-hash
// - 0xe2...8a: pubkey hash
// - compressed txout 2:
// - 0x8009: VLQ-encoded compressed amount for 15000000 (0.15 BTC)
// - 0x00: special script type pay-to-pubkey-hash
// - 0xb8...58: pubkey hash
//
// Example 3:
// From tx in main blockchain:
// Blk 338156, 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620
//
// 0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6
// <><----><><----><-------------------------------------------------->
// | | | \-----------------\ |
// version | \--------\ unspentness |
// height header code compressed txout 22
//
// - version: 1
// - height: 338156
// - header code: 0x10 (2+1 = 3 bytes in unspentness bitmap)
// NOTE: It's +1 since neither bit 1 nor 2 are set, so N-1 is encoded.
// - unspentness: [0x00 0x00 0x10] (bit 20 is set, so output 20+2 = 22 is unspent)
// NOTE: It's +2 since the first two outputs are encoded in the header code
// - compressed txout 22:
// - 0x8ba5b9e763: VLQ-encoded compressed amount for 366875659 (3.66875659 BTC)
// - 0x01: special script type pay-to-script-hash
// - 0x1d...e6: script hash
func deserializeUtxoEntryV0(serialized []byte) (map[uint32]*UtxoEntry, error) {
// Deserialize the version.
//
// NOTE: Ignore version since it is no longer used in the new format.
_, bytesRead := deserializeVLQ(serialized)
offset := bytesRead
if offset >= len(serialized) {
return nil, errDeserialize("unexpected end of data after version")
}
// Deserialize the block height.
blockHeight, bytesRead := deserializeVLQ(serialized[offset:])
offset += bytesRead
if offset >= len(serialized) {
return nil, errDeserialize("unexpected end of data after height")
}
// Deserialize the header code.
code, bytesRead := deserializeVLQ(serialized[offset:])
offset += bytesRead
if offset >= len(serialized) {
return nil, errDeserialize("unexpected end of data after header")
}
// Decode the header code.
//
// Bit 0 indicates whether the containing transaction is a coinbase.
// Bit 1 indicates output 0 is unspent.
// Bit 2 indicates output 1 is unspent.
// Bits 3-x encodes the number of non-zero unspentness bitmap bytes that
// follow. When both output 0 and 1 are spent, it encodes N-1.
isCoinBase := code&0x01 != 0
output0Unspent := code&0x02 != 0
output1Unspent := code&0x04 != 0
numBitmapBytes := code >> 3
if !output0Unspent && !output1Unspent {
numBitmapBytes++
}
// Ensure there are enough bytes left to deserialize the unspentness
// bitmap.
if uint64(len(serialized[offset:])) < numBitmapBytes {
return nil, errDeserialize("unexpected end of data for " +
"unspentness bitmap")
}
// Add sparse output for unspent outputs 0 and 1 as needed based on the
// details provided by the header code.
var outputIndexes []uint32
if output0Unspent {
outputIndexes = append(outputIndexes, 0)
}
if output1Unspent {
outputIndexes = append(outputIndexes, 1)
}
// Decode the unspentness bitmap adding a sparse output for each unspent
// output.
for i := uint32(0); i < uint32(numBitmapBytes); i++ {
unspentBits := serialized[offset]
for j := uint32(0); j < 8; j++ {
if unspentBits&0x01 != 0 {
// The first 2 outputs are encoded via the
// header code, so adjust the output number
// accordingly.
outputNum := 2 + i*8 + j
outputIndexes = append(outputIndexes, outputNum)
}
unspentBits >>= 1
}
offset++
}
// Map to hold all of the converted outputs.
entries := make(map[uint32]*UtxoEntry)
// All entries will need to potentially be marked as a coinbase.
var packedFlags txoFlags
if isCoinBase {
packedFlags |= tfCoinBase
}
// Decode and add all of the utxos.
for i, outputIndex := range outputIndexes {
// Decode the next utxo.
amount, pkScript, bytesRead, err := decodeCompressedTxOut(
serialized[offset:])
if err != nil {
return nil, errDeserialize(fmt.Sprintf("unable to "+
"decode utxo at index %d: %v", i, err))
}
offset += bytesRead
// Create a new utxo entry with the details deserialized above.
entries[outputIndex] = &UtxoEntry{
amount: int64(amount),
pkScript: pkScript,
blockHeight: int32(blockHeight),
packedFlags: packedFlags,
}
}
return entries, nil
}
// upgradeUtxoSetToV2 migrates the utxo set entries from version 1 to 2 in
// batches. It is guaranteed to updated if this returns without failure.
func upgradeUtxoSetToV2(db database.DB, interrupt <-chan struct{}) error {
// Hardcoded bucket names so updates to the global values do not affect
// old upgrades.
var (
v1BucketName = []byte("utxoset")
v2BucketName = []byte("utxosetv2")
)
log.Infof("Upgrading utxo set to v2. This will take a while...")
start := time.Now()
// Create the new utxo set bucket as needed.
err := db.Update(func(dbTx database.Tx) error {
_, err := dbTx.Metadata().CreateBucketIfNotExists(v2BucketName)
return err
})
if err != nil {
return err
}
// doBatch contains the primary logic for upgrading the utxo set from
// version 1 to 2 in batches. This is done because the utxo set can be
// huge and thus attempting to migrate in a single database transaction
// would result in massive memory usage and could potentially crash on
// many systems due to ulimits.
//
// It returns the number of utxos processed.
const maxUtxos = 200000
doBatch := func(dbTx database.Tx) (uint32, error) {
v1Bucket := dbTx.Metadata().Bucket(v1BucketName)
v2Bucket := dbTx.Metadata().Bucket(v2BucketName)
v1Cursor := v1Bucket.Cursor()
// Migrate utxos so long as the max number of utxos for this
// batch has not been exceeded.
var numUtxos uint32
for ok := v1Cursor.First(); ok && numUtxos < maxUtxos; ok =
v1Cursor.Next() {
// Old key was the transaction hash.
oldKey := v1Cursor.Key()
var txHash chainhash.Hash
copy(txHash[:], oldKey)
// Deserialize the old entry which included all utxos
// for the given transaction.
utxos, err := deserializeUtxoEntryV0(v1Cursor.Value())
if err != nil {
return 0, err
}
// Add an entry for each utxo into the new bucket using
// the new format.
for txOutIdx, utxo := range utxos {
reserialized, err := serializeUtxoEntry(utxo)
if err != nil {
return 0, err
}
key := outpointKey(wire.OutPoint{
Hash: txHash,
Index: txOutIdx,
})
err = v2Bucket.Put(*key, reserialized)
// NOTE: The key is intentionally not recycled
// here since the database interface contract
// prohibits modifications. It will be garbage
// collected normally when the database is done
// with it.
if err != nil {
return 0, err
}
}
// Remove old entry.
err = v1Bucket.Delete(oldKey)
if err != nil {
return 0, err
}
numUtxos += uint32(len(utxos))
if interruptRequested(interrupt) {
// No error here so the database transaction
// is not cancelled and therefore outstanding
// work is written to disk.
break
}
}
return numUtxos, nil
}
// Migrate all entries in batches for the reasons mentioned above.
var totalUtxos uint64
for {
var numUtxos uint32
err := db.Update(func(dbTx database.Tx) error {
var err error
numUtxos, err = doBatch(dbTx)
return err
})
if err != nil {
return err
}
if interruptRequested(interrupt) {
return errInterruptRequested
}
if numUtxos == 0 {
break
}
totalUtxos += uint64(numUtxos)
log.Infof("Migrated %d utxos (%d total)", numUtxos, totalUtxos)
}
// Remove the old bucket and update the utxo set version once it has
// been fully migrated.
err = db.Update(func(dbTx database.Tx) error {
err := dbTx.Metadata().DeleteBucket(v1BucketName)
if err != nil {
return err
}
return dbPutVersion(dbTx, utxoSetVersionKeyName, 2)
})
if err != nil {
return err
}
seconds := int64(time.Since(start) / time.Second)
log.Infof("Done upgrading utxo set. Total utxos: %d in %d seconds",
totalUtxos, seconds)
return nil
}
// maybeUpgradeDbBuckets checks the database version of the buckets used by this
// package and performs any needed upgrades to bring them to the latest version.
//
// All buckets used by this package are guaranteed to be the latest version if
// this function returns without error.
func (b *BlockChain) maybeUpgradeDbBuckets(interrupt <-chan struct{}) error {
// Load or create bucket versions as needed.
var utxoSetVersion uint32
err := b.db.Update(func(dbTx database.Tx) error {
// Load the utxo set version from the database or create it and
// initialize it to version 1 if it doesn't exist.
var err error
utxoSetVersion, err = dbFetchOrCreateVersion(dbTx,
utxoSetVersionKeyName, 1)
return err
})
if err != nil {
return err
}
// Update the utxo set to v2 if needed.
if utxoSetVersion < 2 {
if err := upgradeUtxoSetToV2(b.db, interrupt); err != nil {
return err
}
}
return nil
}

116
blockchain/upgrade_test.go Normal file
View file

@ -0,0 +1,116 @@
// Copyright (c) 2015-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
package blockchain
import (
"reflect"
"testing"
)
// TestDeserializeUtxoEntryV0 ensures deserializing unspent trasaction output
// entries from the legacy version 0 format works as expected.
func TestDeserializeUtxoEntryV0(t *testing.T) {
tests := []struct {
name string
entries map[uint32]*UtxoEntry
serialized []byte
}{
// From tx in main blockchain:
// 0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098
{
name: "Only output 0, coinbase",
entries: map[uint32]*UtxoEntry{
0: {
amount: 5000000000,
pkScript: hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
blockHeight: 1,
packedFlags: tfCoinBase,
},
},
serialized: hexToBytes("010103320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"),
},
// From tx in main blockchain:
// 8131ffb0a2c945ecaf9b9063e59558784f9c3a74741ce6ae2a18d0571dac15bb
{
name: "Only output 1, not coinbase",
entries: map[uint32]*UtxoEntry{
1: {
amount: 1000000,
pkScript: hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
blockHeight: 100001,
packedFlags: 0,
},
},
serialized: hexToBytes("01858c21040700ee8bd501094a7d5ca318da2506de35e1cb025ddc"),
},
// Adapted from tx in main blockchain:
// df3f3f442d9699857f7f49de4ff0b5d0f3448bec31cdc7b5bf6d25f2abd637d5
{
name: "Only output 2, coinbase",
entries: map[uint32]*UtxoEntry{
2: {
amount: 100937281,
pkScript: hexToBytes("76a914da33f77cee27c2a975ed5124d7e4f7f97513510188ac"),
blockHeight: 99004,
packedFlags: tfCoinBase,
},
},
serialized: hexToBytes("0185843c010182b095bf4100da33f77cee27c2a975ed5124d7e4f7f975135101"),
},
// Adapted from tx in main blockchain:
// 4a16969aa4764dd7507fc1de7f0baa4850a246de90c45e59a3207f9a26b5036f
{
name: "outputs 0 and 2 not coinbase",
entries: map[uint32]*UtxoEntry{
0: {
amount: 20000000,
pkScript: hexToBytes("76a914e2ccd6ec7c6e2e581349c77e067385fa8236bf8a88ac"),
blockHeight: 113931,
packedFlags: 0,
},
2: {
amount: 15000000,
pkScript: hexToBytes("76a914b8025be1b3efc63b0ad48e7f9f10e87544528d5888ac"),
blockHeight: 113931,
packedFlags: 0,
},
},
serialized: hexToBytes("0185f90b0a011200e2ccd6ec7c6e2e581349c77e067385fa8236bf8a800900b8025be1b3efc63b0ad48e7f9f10e87544528d58"),
},
// Adapted from tx in main blockchain:
// 1b02d1c8cfef60a189017b9a420c682cf4a0028175f2f563209e4ff61c8c3620
{
name: "Only output 22, not coinbase",
entries: map[uint32]*UtxoEntry{
22: {
amount: 366875659,
pkScript: hexToBytes("a9141dd46a006572d820e448e12d2bbb38640bc718e687"),
blockHeight: 338156,
packedFlags: 0,
},
},
serialized: hexToBytes("0193d06c100000108ba5b9e763011dd46a006572d820e448e12d2bbb38640bc718e6"),
},
}
for i, test := range tests {
// Deserialize to map of utxos keyed by the output index.
entries, err := deserializeUtxoEntryV0(test.serialized)
if err != nil {
t.Errorf("deserializeUtxoEntryV0 #%d (%s) unexpected "+
"error: %v", i, test.name, err)
continue
}
// Ensure the deserialized entry has the same properties as the
// ones in the test entry.
if !reflect.DeepEqual(entries, test.entries) {
t.Errorf("deserializeUtxoEntryV0 #%d (%s) unexpected "+
"entries: got %v, want %v", i, test.name,
entries, test.entries)
continue
}
}
}

622
blockchain/utxoviewpoint.go
View file

@ -10,179 +10,104 @@ import (
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/database"
"github.com/btcsuite/btcd/txscript"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
)
// utxoOutput houses details about an individual unspent transaction output such
// as whether or not it is spent, its public key script, and how much it pays.
//
// Standard public key scripts are stored in the database using a compressed
// format. Since the vast majority of scripts are of the standard form, a fairly
// significant savings is achieved by discarding the portions of the standard
// scripts that can be reconstructed.
//
// Also, since it is common for only a specific output in a given utxo entry to
// be referenced from a redeeming transaction, the script and amount for a given
// output is not uncompressed until the first time it is accessed. This
// provides a mechanism to avoid the overhead of needlessly uncompressing all
// outputs for a given utxo entry at the time of load.
type utxoOutput struct {
spent bool // Output is spent.
compressed bool // The amount and public key script are compressed.
amount int64 // The amount of the output.
pkScript []byte // The public key script for the output.
}
// txoFlags is a bitmask defining additional information and state for a
// transaction output in a utxo view.
type txoFlags uint8
// maybeDecompress decompresses the amount and public key script fields of the
// utxo and marks it decompressed if needed.
func (o *utxoOutput) maybeDecompress(version int32) {
// Nothing to do if it's not compressed.
if !o.compressed {
return
}
const (
// tfCoinBase indicates that a txout was contained in a coinbase tx.
tfCoinBase txoFlags = 1 << iota
o.amount = int64(decompressTxOutAmount(uint64(o.amount)))
o.pkScript = decompressScript(o.pkScript, version)
o.compressed = false
}
// tfSpent indicates that a txout is spent.
tfSpent
// UtxoEntry contains contextual information about an unspent transaction such
// as whether or not it is a coinbase transaction, which block it was found in,
// and the spent status of its outputs.
// tfModified indicates that a txout has been modified since it was
// loaded.
tfModified
)
// UtxoEntry houses details about an individual transaction output in a utxo
// view such as whether or not it was contained in a coinbase tx, the height of
// the block that contains the tx, whether or not it is spent, its public key
// script, and how much it pays.
type UtxoEntry struct {
modified bool // Entry changed since load.
version int32 // The version of this tx.
isCoinBase bool // Whether entry is a coinbase tx.
blockHeight int32 // Height of block containing tx.
sparseOutputs map[uint32]*utxoOutput // Sparse map of unspent outputs.
// NOTE: Additions, deletions, or modifications to the order of the
// definitions in this struct should not be changed without considering
// how it affects alignment on 64-bit platforms. The current order is
// specifically crafted to result in minimal padding. There will be a
// lot of these in memory, so a few extra bytes of padding adds up.
amount int64
pkScript []byte // The public key script for the output.
blockHeight int32 // Height of block containing tx.
// packedFlags contains additional info about output such as whether it
// is a coinbase, whether it is spent, and whether it has been modified
// since it was loaded. This approach is used in order to reduce memory
// usage since there will be a lot of these in memory.
packedFlags txoFlags
}
// Version returns the version of the transaction the utxo represents.
func (entry *UtxoEntry) Version() int32 {
return entry.version
// isModified returns whether or not the output has been modified since it was
// loaded.
func (entry *UtxoEntry) isModified() bool {
return entry.packedFlags&tfModified == tfModified
}
// IsCoinBase returns whether or not the transaction the utxo entry represents
// is a coinbase.
// IsCoinBase returns whether or not the output was contained in a coinbase
// transaction.
func (entry *UtxoEntry) IsCoinBase() bool {
return entry.isCoinBase
return entry.packedFlags&tfCoinBase == tfCoinBase
}
// BlockHeight returns the height of the block containing the transaction the
// utxo entry represents.
// BlockHeight returns the height of the block containing the output.
func (entry *UtxoEntry) BlockHeight() int32 {
return entry.blockHeight
}
// IsOutputSpent returns whether or not the provided output index has been
// spent based upon the current state of the unspent transaction output view
// the entry was obtained from.
//
// Returns true if the output index references an output that does not exist
// either due to it being invalid or because the output is not part of the view
// due to previously being spent/pruned.
func (entry *UtxoEntry) IsOutputSpent(outputIndex uint32) bool {
output, ok := entry.sparseOutputs[outputIndex]
if !ok {
return true
}
return output.spent
// IsSpent returns whether or not the output has been spent based upon the
// current state of the unspent transaction output view it was obtained from.
func (entry *UtxoEntry) IsSpent() bool {
return entry.packedFlags&tfSpent == tfSpent
}
// SpendOutput marks the output at the provided index as spent. Specifying an
// output index that does not exist will not have any effect.
func (entry *UtxoEntry) SpendOutput(outputIndex uint32) {
output, ok := entry.sparseOutputs[outputIndex]
if !ok {
return
}
// Spend marks the output as spent. Spending an output that is already spent
// has no effect.
func (entry *UtxoEntry) Spend() {
// Nothing to do if the output is already spent.
if output.spent {
if entry.IsSpent() {
return
}
entry.modified = true
output.spent = true
// Mark the output as spent and modified.
entry.packedFlags |= tfSpent | tfModified
}
// IsFullySpent returns whether or not the transaction the utxo entry represents
// is fully spent.
func (entry *UtxoEntry) IsFullySpent() bool {
// The entry is not fully spent if any of the outputs are unspent.
for _, output := range entry.sparseOutputs {
if !output.spent {
return false
}
}
return true
// Amount returns the amount of the output.
func (entry *UtxoEntry) Amount() int64 {
return entry.amount
}
// AmountByIndex returns the amount of the provided output index.
//
// Returns 0 if the output index references an output that does not exist
// either due to it being invalid or because the output is not part of the view
// due to previously being spent/pruned.
func (entry *UtxoEntry) AmountByIndex(outputIndex uint32) int64 {
output, ok := entry.sparseOutputs[outputIndex]
if !ok {
return 0
}
// Ensure the output is decompressed before returning the amount.
output.maybeDecompress(entry.version)
return output.amount
// PkScript returns the public key script for the output.
func (entry *UtxoEntry) PkScript() []byte {
return entry.pkScript
}
// PkScriptByIndex returns the public key script for the provided output index.
//
// Returns nil if the output index references an output that does not exist
// either due to it being invalid or because the output is not part of the view
// due to previously being spent/pruned.
func (entry *UtxoEntry) PkScriptByIndex(outputIndex uint32) []byte {
output, ok := entry.sparseOutputs[outputIndex]
if !ok {
return nil
}
// Ensure the output is decompressed before returning the script.
output.maybeDecompress(entry.version)
return output.pkScript
}
// Clone returns a deep copy of the utxo entry.
// Clone returns a shallow copy of the utxo entry.
func (entry *UtxoEntry) Clone() *UtxoEntry {
if entry == nil {
return nil
}
newEntry := &UtxoEntry{
version: entry.version,
isCoinBase: entry.isCoinBase,
blockHeight: entry.blockHeight,
sparseOutputs: make(map[uint32]*utxoOutput),
}
for outputIndex, output := range entry.sparseOutputs {
newEntry.sparseOutputs[outputIndex] = &utxoOutput{
spent: output.spent,
compressed: output.compressed,
amount: output.amount,
pkScript: output.pkScript,
}
}
return newEntry
}
// newUtxoEntry returns a new unspent transaction output entry with the provided
// coinbase flag and block height ready to have unspent outputs added.
func newUtxoEntry(version int32, isCoinBase bool, blockHeight int32) *UtxoEntry {
return &UtxoEntry{
version: version,
isCoinBase: isCoinBase,
blockHeight: blockHeight,
sparseOutputs: make(map[uint32]*utxoOutput),
amount: entry.amount,
pkScript: entry.pkScript,
blockHeight: entry.blockHeight,
packedFlags: entry.packedFlags,
}
}
@ -194,7 +119,7 @@ func newUtxoEntry(version int32, isCoinBase bool, blockHeight int32) *UtxoEntry
// The unspent outputs are needed by other transactions for things such as
// script validation and double spend prevention.
type UtxoViewpoint struct {
entries map[chainhash.Hash]*UtxoEntry
entries map[wire.OutPoint]*UtxoEntry
bestHash chainhash.Hash
}
@ -210,17 +135,60 @@ func (view *UtxoViewpoint) SetBestHash(hash *chainhash.Hash) {
view.bestHash = *hash
}
// LookupEntry returns information about a given transaction according to the
// current state of the view. It will return nil if the passed transaction
// hash does not exist in the view or is otherwise not available such as when
// it has been disconnected during a reorg.
func (view *UtxoViewpoint) LookupEntry(txHash *chainhash.Hash) *UtxoEntry {
entry, ok := view.entries[*txHash]
if !ok {
return nil
// LookupEntry returns information about a given transaction output according to
// the current state of the view. It will return nil if the passed output does
// not exist in the view or is otherwise not available such as when it has been
// disconnected during a reorg.
func (view *UtxoViewpoint) LookupEntry(outpoint wire.OutPoint) *UtxoEntry {
return view.entries[outpoint]
}
// addTxOut adds the specified output to the view if it is not provably
// unspendable. When the view already has an entry for the output, it will be
// marked unspent. All fields will be updated for existing entries since it's
// possible it has changed during a reorg.
func (view *UtxoViewpoint) addTxOut(outpoint wire.OutPoint, txOut *wire.TxOut, isCoinBase bool, blockHeight int32) {
// Don't add provably unspendable outputs.
if txscript.IsUnspendable(txOut.PkScript) {
return
}
return entry
// Update existing entries. All fields are updated because it's
// possible (although extremely unlikely) that the existing entry is
// being replaced by a different transaction with the same hash. This
// is allowed so long as the previous transaction is fully spent.
entry := view.LookupEntry(outpoint)
if entry == nil {
entry = new(UtxoEntry)
view.entries[outpoint] = entry
}
entry.amount = txOut.Value
entry.pkScript = txOut.PkScript
entry.blockHeight = blockHeight
entry.packedFlags = tfModified
if isCoinBase {
entry.packedFlags |= tfCoinBase
}
}
// AddTxOut adds the specified output of the passed transaction to the view if
// it exists and is not provably unspendable. When the view already has an
// entry for the output, it will be marked unspent. All fields will be updated
// for existing entries since it's possible it has changed during a reorg.
func (view *UtxoViewpoint) AddTxOut(tx *btcutil.Tx, txOutIdx uint32, blockHeight int32) {
// Can't add an output for an out of bounds index.
if txOutIdx >= uint32(len(tx.MsgTx().TxOut)) {
return
}
// Update existing entries. All fields are updated because it's
// possible (although extremely unlikely) that the existing entry is
// being replaced by a different transaction with the same hash. This
// is allowed so long as the previous transaction is fully spent.
prevOut := wire.OutPoint{Hash: *tx.Hash(), Index: txOutIdx}
txOut := tx.MsgTx().TxOut[txOutIdx]
view.addTxOut(prevOut, txOut, IsCoinBase(tx), blockHeight)
}
// AddTxOuts adds all outputs in the passed transaction which are not provably
@ -228,45 +196,18 @@ func (view *UtxoViewpoint) LookupEntry(txHash *chainhash.Hash) *UtxoEntry {
// outputs, they are simply marked unspent. All fields will be updated for
// existing entries since it's possible it has changed during a reorg.
func (view *UtxoViewpoint) AddTxOuts(tx *btcutil.Tx, blockHeight int32) {
// When there are not already any utxos associated with the transaction,
// add a new entry for it to the view.
entry := view.LookupEntry(tx.Hash())
if entry == nil {
entry = newUtxoEntry(tx.MsgTx().Version, IsCoinBase(tx),
blockHeight)
view.entries[*tx.Hash()] = entry
} else {
entry.blockHeight = blockHeight
}
entry.modified = true
// Loop all of the transaction outputs and add those which are not
// provably unspendable.
isCoinBase := IsCoinBase(tx)
prevOut := wire.OutPoint{Hash: *tx.Hash()}
for txOutIdx, txOut := range tx.MsgTx().TxOut {
if txscript.IsUnspendable(txOut.PkScript) {
continue
}
// Update existing entries. All fields are updated because it's
// possible (although extremely unlikely) that the existing
// entry is being replaced by a different transaction with the
// same hash. This is allowed so long as the previous
// transaction is fully spent.
if output, ok := entry.sparseOutputs[uint32(txOutIdx)]; ok {
output.spent = false
output.compressed = false
output.amount = txOut.Value
output.pkScript = txOut.PkScript
continue
}
// Add the unspent transaction output.
entry.sparseOutputs[uint32(txOutIdx)] = &utxoOutput{
spent: false,
compressed: false,
amount: txOut.Value,
pkScript: txOut.PkScript,
}
prevOut.Index = uint32(txOutIdx)
view.addTxOut(prevOut, txOut, isCoinBase, blockHeight)
}
}
@ -287,39 +228,30 @@ func (view *UtxoViewpoint) connectTransaction(tx *btcutil.Tx, blockHeight int32,
// if a slice was provided for the spent txout details, append an entry
// to it.
for _, txIn := range tx.MsgTx().TxIn {
originIndex := txIn.PreviousOutPoint.Index
entry := view.entries[txIn.PreviousOutPoint.Hash]
// Ensure the referenced utxo exists in the view. This should
// never happen unless there is a bug is introduced in the code.
entry := view.entries[txIn.PreviousOutPoint]
if entry == nil {
return AssertError(fmt.Sprintf("view missing input %v",
txIn.PreviousOutPoint))
}
entry.SpendOutput(originIndex)
// Don't create the stxo details if not requested.
if stxos == nil {
continue
// Only create the stxo details if requested.
if stxos != nil {
// Populate the stxo details using the utxo entry.
var stxo = spentTxOut{
amount: entry.Amount(),
pkScript: entry.PkScript(),
height: entry.BlockHeight(),
isCoinBase: entry.IsCoinBase(),
}
*stxos = append(*stxos, stxo)
}
// Populate the stxo details using the utxo entry. When the
// transaction is fully spent, set the additional stxo fields
// accordingly since those details will no longer be available
// in the utxo set.
var stxo = spentTxOut{
compressed: false,
version: entry.Version(),
amount: entry.AmountByIndex(originIndex),
pkScript: entry.PkScriptByIndex(originIndex),
}
if entry.IsFullySpent() {
stxo.height = entry.BlockHeight()
stxo.isCoinBase = entry.IsCoinBase()
}
// Append the entry to the provided spent txouts slice.
*stxos = append(*stxos, stxo)
// Mark the entry as spent. This is not done until after the
// relevant details have been accessed since spending it might
// clear the fields from memory in the future.
entry.Spend()
}
// Add the transaction's outputs as available utxos.
@ -346,11 +278,37 @@ func (view *UtxoViewpoint) connectTransactions(block *btcutil.Block, stxos *[]sp
return nil
}
// fetchEntryByHash attempts to find any available utxo for the given hash by
// searching the entire set of possible outputs for the given hash. It checks
// the view first and then falls back to the database if needed.
func (view *UtxoViewpoint) fetchEntryByHash(db database.DB, hash *chainhash.Hash) (*UtxoEntry, error) {
// First attempt to find a utxo with the provided hash in the view.
prevOut := wire.OutPoint{Hash: *hash}
for idx := uint32(0); idx < MaxOutputsPerBlock; idx++ {
prevOut.Index = idx
entry := view.LookupEntry(prevOut)
if entry != nil {
return entry, nil
}
}
// Check the database since it doesn't exist in the view. This will
// often by the case since only specifically referenced utxos are loaded
// into the view.
var entry *UtxoEntry
err := db.View(func(dbTx database.Tx) error {
var err error
entry, err = dbFetchUtxoEntryByHash(dbTx, hash)
return err
})
return entry, err
}
// disconnectTransactions updates the view by removing all of the transactions
// created by the passed block, restoring all utxos the transactions spent by
// using the provided spent txo information, and setting the best hash for the
// view to the block before the passed block.
func (view *UtxoViewpoint) disconnectTransactions(block *btcutil.Block, stxos []spentTxOut) error {
func (view *UtxoViewpoint) disconnectTransactions(db database.DB, block *btcutil.Block, stxos []spentTxOut) error {
// Sanity check the correct number of stxos are provided.
if len(stxos) != countSpentOutputs(block) {
return AssertError("disconnectTransactions called with bad " +
@ -365,25 +323,52 @@ func (view *UtxoViewpoint) disconnectTransactions(block *btcutil.Block, stxos []
for txIdx := len(transactions) - 1; txIdx > -1; txIdx-- {
tx := transactions[txIdx]
// Clear this transaction from the view if it already exists or
// create a new empty entry for when it does not. This is done
// because the code relies on its existence in the view in order
// to signal modifications have happened.
isCoinbase := txIdx == 0
entry := view.entries[*tx.Hash()]
if entry == nil {
entry = newUtxoEntry(tx.MsgTx().Version, isCoinbase,
block.Height())
view.entries[*tx.Hash()] = entry
// All entries will need to potentially be marked as a coinbase.
var packedFlags txoFlags
isCoinBase := txIdx == 0
if isCoinBase {
packedFlags |= tfCoinBase
}
// Mark all of the spendable outputs originally created by the
// transaction as spent. It is instructive to note that while
// the outputs aren't actually being spent here, rather they no
// longer exist, since a pruned utxo set is used, there is no
// practical difference between a utxo that does not exist and
// one that has been spent.
//
// When the utxo does not already exist in the view, add an
// entry for it and then mark it spent. This is done because
// the code relies on its existence in the view in order to
// signal modifications have happened.
txHash := tx.Hash()
prevOut := wire.OutPoint{Hash: *txHash}
for txOutIdx, txOut := range tx.MsgTx().TxOut {
if txscript.IsUnspendable(txOut.PkScript) {
continue
}
prevOut.Index = uint32(txOutIdx)
entry := view.entries[prevOut]
if entry == nil {
entry = &UtxoEntry{
amount: txOut.Value,
pkScript: txOut.PkScript,
blockHeight: block.Height(),
packedFlags: packedFlags,
}
view.entries[prevOut] = entry
}
entry.Spend()
}
entry.modified = true
entry.sparseOutputs = make(map[uint32]*utxoOutput)
// Loop backwards through all of the transaction inputs (except
// for the coinbase which has no inputs) and unspend the
// referenced txos. This is necessary to match the order of the
// spent txout entries.
if isCoinbase {
if isCoinBase {
continue
}
for txInIdx := len(tx.MsgTx().TxIn) - 1; txInIdx > -1; txInIdx-- {
@ -393,40 +378,57 @@ func (view *UtxoViewpoint) disconnectTransactions(block *btcutil.Block, stxos []
stxoIdx--
// When there is not already an entry for the referenced
// transaction in the view, it means it was fully spent,
// output in the view, it means it was previously spent,
// so create a new utxo entry in order to resurrect it.
txIn := tx.MsgTx().TxIn[txInIdx]
originHash := &txIn.PreviousOutPoint.Hash
originIndex := txIn.PreviousOutPoint.Index
entry := view.entries[*originHash]
originOut := &tx.MsgTx().TxIn[txInIdx].PreviousOutPoint
entry := view.entries[*originOut]
if entry == nil {
entry = newUtxoEntry(stxo.version,
stxo.isCoinBase, stxo.height)
view.entries[*originHash] = entry
entry = new(UtxoEntry)
view.entries[*originOut] = entry
}
// Mark the entry as modified since it is either new
// or will be changed below.
entry.modified = true
// Restore the specific utxo using the stxo data from
// the spend journal if it doesn't already exist in the
// view.
output, ok := entry.sparseOutputs[originIndex]
if !ok {
// Add the unspent transaction output.
entry.sparseOutputs[originIndex] = &utxoOutput{
spent: false,
compressed: stxo.compressed,
amount: stxo.amount,
pkScript: stxo.pkScript,
// The legacy v1 spend journal format only stored the
// coinbase flag and height when the output was the last
// unspent output of the transaction. As a result, when
// the information is missing, search for it by scanning
// all possible outputs of the transaction since it must
// be in one of them.
//
// It should be noted that this is quite inefficient,
// but it realistically will almost never run since all
// new entries include the information for all outputs
// and thus the only way this will be hit is if a long
// enough reorg happens such that a block with the old
// spend data is being disconnected. The probability of
// that in practice is extremely low to begin with and
// becomes vanishingly small the more new blocks are
// connected. In the case of a fresh database that has
// only ever run with the new v2 format, this code path
// will never run.
if stxo.height == 0 {
utxo, err := view.fetchEntryByHash(db, txHash)
if err != nil {
return err
}
continue
if utxo == nil {
return AssertError(fmt.Sprintf("unable "+
"to resurrect legacy stxo %v",
*originOut))
}
stxo.height = utxo.BlockHeight()
stxo.isCoinBase = utxo.IsCoinBase()
}
// Mark the existing referenced transaction output as
// unspent.
output.spent = false
// Restore the utxo using the stxo data from the spend
// journal and mark it as modified.
entry.amount = stxo.amount
entry.pkScript = stxo.pkScript
entry.blockHeight = stxo.height
entry.packedFlags = tfModified
if stxo.isCoinBase {
entry.packedFlags |= tfCoinBase
}
}
}
@ -436,88 +438,94 @@ func (view *UtxoViewpoint) disconnectTransactions(block *btcutil.Block, stxos []
return nil
}
// RemoveEntry removes the given transaction output from the current state of
// the view. It will have no effect if the passed output does not exist in the
// view.
func (view *UtxoViewpoint) RemoveEntry(outpoint wire.OutPoint) {
delete(view.entries, outpoint)
}
// Entries returns the underlying map that stores of all the utxo entries.
func (view *UtxoViewpoint) Entries() map[chainhash.Hash]*UtxoEntry {
func (view *UtxoViewpoint) Entries() map[wire.OutPoint]*UtxoEntry {
return view.entries
}
// commit prunes all entries marked modified that are now fully spent and marks
// all entries as unmodified.
func (view *UtxoViewpoint) commit() {
for txHash, entry := range view.entries {
if entry == nil || (entry.modified && entry.IsFullySpent()) {
delete(view.entries, txHash)
for outpoint, entry := range view.entries {
if entry == nil || (entry.isModified() && entry.IsSpent()) {
delete(view.entries, outpoint)
continue
}
entry.modified = false
entry.packedFlags ^= tfModified
}
}
// fetchUtxosMain fetches unspent transaction output data about the provided
// set of transactions from the point of view of the end of the main chain at
// the time of the call.
// set of outpoints from the point of view of the end of the main chain at the
// time of the call.
//
// Upon completion of this function, the view will contain an entry for each
// requested transaction. Fully spent transactions, or those which otherwise
// don't exist, will result in a nil entry in the view.
func (view *UtxoViewpoint) fetchUtxosMain(db database.DB, txSet map[chainhash.Hash]struct{}) error {
// Nothing to do if there are no requested hashes.
if len(txSet) == 0 {
// requested outpoint. Spent outputs, or those which otherwise don't exist,
// will result in a nil entry in the view.
func (view *UtxoViewpoint) fetchUtxosMain(db database.DB, outpoints map[wire.OutPoint]struct{}) error {
// Nothing to do if there are no requested outputs.
if len(outpoints) == 0 {
return nil
}
// Load the unspent transaction output information for the requested set
// of transactions from the point of view of the end of the main chain.
// Load the requested set of unspent transaction outputs from the point
// of view of the end of the main chain.
//
// NOTE: Missing entries are not considered an error here and instead
// will result in nil entries in the view. This is intentionally done
// since other code uses the presence of an entry in the store as a way
// to optimize spend and unspend updates to apply only to the specific
// utxos that the caller needs access to.
// so other code can use the presence of an entry in the store as a way
// to unnecessarily avoid attempting to reload it from the database.
return db.View(func(dbTx database.Tx) error {
for hash := range txSet {
hashCopy := hash
entry, err := dbFetchUtxoEntry(dbTx, &hashCopy)
for outpoint := range outpoints {
entry, err := dbFetchUtxoEntry(dbTx, outpoint)
if err != nil {
return err
}
view.entries[hash] = entry
view.entries[outpoint] = entry
}
return nil
})
}
// fetchUtxos loads utxo details about provided set of transaction hashes into
// the view from the database as needed unless they already exist in the view in
// which case they are ignored.
func (view *UtxoViewpoint) fetchUtxos(db database.DB, txSet map[chainhash.Hash]struct{}) error {
// Nothing to do if there are no requested hashes.
if len(txSet) == 0 {
// fetchUtxos loads the unspent transaction outputs for the provided set of
// outputs into the view from the database as needed unless they already exist
// in the view in which case they are ignored.
func (view *UtxoViewpoint) fetchUtxos(db database.DB, outpoints map[wire.OutPoint]struct{}) error {
// Nothing to do if there are no requested outputs.
if len(outpoints) == 0 {
return nil
}
// Filter entries that are already in the view.
txNeededSet := make(map[chainhash.Hash]struct{})
for hash := range txSet {
neededSet := make(map[wire.OutPoint]struct{})
for outpoint := range outpoints {
// Already loaded into the current view.
if _, ok := view.entries[hash]; ok {
if _, ok := view.entries[outpoint]; ok {
continue
}
txNeededSet[hash] = struct{}{}
neededSet[outpoint] = struct{}{}
}
// Request the input utxos from the database.
return view.fetchUtxosMain(db, txNeededSet)
return view.fetchUtxosMain(db, neededSet)
}
// fetchInputUtxos loads utxo details about the input transactions referenced
// by the transactions in the given block into the view from the database as
// needed. In particular, referenced entries that are earlier in the block are
// added to the view and entries that are already in the view are not modified.
// fetchInputUtxos loads the unspent transaction outputs for the inputs
// referenced by the transactions in the given block into the view from the
// database as needed. In particular, referenced entries that are earlier in
// the block are added to the view and entries that are already in the view are
// not modified.
func (view *UtxoViewpoint) fetchInputUtxos(db database.DB, block *btcutil.Block) error {
// Build a map of in-flight transactions because some of the inputs in
// this block could be referencing other transactions earlier in this
@ -531,7 +539,7 @@ func (view *UtxoViewpoint) fetchInputUtxos(db database.DB, block *btcutil.Block)
// Loop through all of the transaction inputs (except for the coinbase
// which has no inputs) collecting them into sets of what is needed and
// what is already known (in-flight).
txNeededSet := make(map[chainhash.Hash]struct{})
neededSet := make(map[wire.OutPoint]struct{})
for i, tx := range transactions[1:] {
for _, txIn := range tx.MsgTx().TxIn {
// It is acceptable for a transaction input to reference
@ -556,72 +564,74 @@ func (view *UtxoViewpoint) fetchInputUtxos(db database.DB, block *btcutil.Block)
// Don't request entries that are already in the view
// from the database.
if _, ok := view.entries[*originHash]; ok {
if _, ok := view.entries[txIn.PreviousOutPoint]; ok {
continue
}
txNeededSet[*originHash] = struct{}{}
neededSet[txIn.PreviousOutPoint] = struct{}{}
}
}
// Request the input utxos from the database.
return view.fetchUtxosMain(db, txNeededSet)
return view.fetchUtxosMain(db, neededSet)
}
// NewUtxoViewpoint returns a new empty unspent transaction output view.
func NewUtxoViewpoint() *UtxoViewpoint {
return &UtxoViewpoint{
entries: make(map[chainhash.Hash]*UtxoEntry),
entries: make(map[wire.OutPoint]*UtxoEntry),
}
}
// FetchUtxoView loads utxo details about the input transactions referenced by
// FetchUtxoView loads unspent transaction outputs for the inputs referenced by
// the passed transaction from the point of view of the end of the main chain.
// It also attempts to fetch the utxo details for the transaction itself so the
// returned view can be examined for duplicate unspent transaction outputs.
// It also attempts to fetch the utxos for the outputs of the transaction itself
// so the returned view can be examined for duplicate transactions.
//
// This function is safe for concurrent access however the returned view is NOT.
func (b *BlockChain) FetchUtxoView(tx *btcutil.Tx) (*UtxoViewpoint, error) {
b.chainLock.RLock()
defer b.chainLock.RUnlock()
// Create a set of needed transactions based on those referenced by the
// inputs of the passed transaction. Also, add the passed transaction
// itself as a way for the caller to detect duplicates that are not
// fully spent.
txNeededSet := make(map[chainhash.Hash]struct{})
txNeededSet[*tx.Hash()] = struct{}{}
// Create a set of needed outputs based on those referenced by the
// inputs of the passed transaction and the outputs of the transaction
// itself.
neededSet := make(map[wire.OutPoint]struct{})
prevOut := wire.OutPoint{Hash: *tx.Hash()}
for txOutIdx := range tx.MsgTx().TxOut {
prevOut.Index = uint32(txOutIdx)
neededSet[prevOut] = struct{}{}
}
if !IsCoinBase(tx) {
for _, txIn := range tx.MsgTx().TxIn {
txNeededSet[txIn.PreviousOutPoint.Hash] = struct{}{}
neededSet[txIn.PreviousOutPoint] = struct{}{}
}
}
// Request the utxos from the point of view of the end of the main
// chain.
view := NewUtxoViewpoint()
err := view.fetchUtxosMain(b.db, txNeededSet)
b.chainLock.RLock()
err := view.fetchUtxosMain(b.db, neededSet)
b.chainLock.RUnlock()
return view, err
}
// FetchUtxoEntry loads and returns the unspent transaction output entry for the
// passed hash from the point of view of the end of the main chain.
// FetchUtxoEntry loads and returns the requested unspent transaction output
// from the point of view of the end of the main chain.
//
// NOTE: Requesting a hash for which there is no data will NOT return an error.
// Instead both the entry and the error will be nil. This is done to allow
// pruning of fully spent transactions. In practice this means the caller must
// check if the returned entry is nil before invoking methods on it.
// NOTE: Requesting an output for which there is no data will NOT return an
// error. Instead both the entry and the error will be nil. This is done to
// allow pruning of spent transaction outputs. In practice this means the
// caller must check if the returned entry is nil before invoking methods on it.
//
// This function is safe for concurrent access however the returned entry (if
// any) is NOT.
func (b *BlockChain) FetchUtxoEntry(txHash *chainhash.Hash) (*UtxoEntry, error) {
func (b *BlockChain) FetchUtxoEntry(outpoint wire.OutPoint) (*UtxoEntry, error) {
b.chainLock.RLock()
defer b.chainLock.RUnlock()
var entry *UtxoEntry
err := b.db.View(func(dbTx database.Tx) error {
var err error
entry, err = dbFetchUtxoEntry(dbTx, txHash)
entry, err = dbFetchUtxoEntry(dbTx, outpoint)
return err
})
if err != nil {

52
blockchain/validate.go
View file

@ -286,8 +286,7 @@ func CheckTransactionSanity(tx *btcutil.Tx) error {
// Previous transaction outputs referenced by the inputs to this
// transaction must not be null.
for _, txIn := range msgTx.TxIn {
prevOut := &txIn.PreviousOutPoint
if isNullOutpoint(prevOut) {
if isNullOutpoint(&txIn.PreviousOutPoint) {
return ruleError(ErrBadTxInput, "transaction "+
"input refers to previous output that "+
"is null")
@ -385,10 +384,8 @@ func CountP2SHSigOps(tx *btcutil.Tx, isCoinBaseTx bool, utxoView *UtxoViewpoint)
totalSigOps := 0
for txInIndex, txIn := range msgTx.TxIn {
// Ensure the referenced input transaction is available.
originTxHash := &txIn.PreviousOutPoint.Hash
originTxIndex := txIn.PreviousOutPoint.Index
txEntry := utxoView.LookupEntry(originTxHash)
if txEntry == nil || txEntry.IsOutputSpent(originTxIndex) {
utxo := utxoView.LookupEntry(txIn.PreviousOutPoint)
if utxo == nil || utxo.IsSpent() {
str := fmt.Sprintf("output %v referenced from "+
"transaction %s:%d either does not exist or "+
"has already been spent", txIn.PreviousOutPoint,
@ -398,7 +395,7 @@ func CountP2SHSigOps(tx *btcutil.Tx, isCoinBaseTx bool, utxoView *UtxoViewpoint)
// We're only interested in pay-to-script-hash types, so skip
// this input if it's not one.
pkScript := txEntry.PkScriptByIndex(originTxIndex)
pkScript := utxo.PkScript()
if !txscript.IsPayToScriptHash(pkScript) {
continue
}
@ -827,16 +824,21 @@ func (b *BlockChain) checkBlockContext(block *btcutil.Block, prevNode *blockNode
// duplicated to effectively revert the overwritten transactions to a single
// confirmation thereby making them vulnerable to a double spend.
//
// For more details, see https://en.bitcoin.it/wiki/BIP_0030 and
// For more details, see
// https://github.com/bitcoin/bips/blob/master/bip-0030.mediawiki and
// http://r6.ca/blog/20120206T005236Z.html.
//
// This function MUST be called with the chain state lock held (for reads).
func (b *BlockChain) checkBIP0030(node *blockNode, block *btcutil.Block, view *UtxoViewpoint) error {
// Fetch utxo details for all of the transactions in this block.
// Typically, there will not be any utxos for any of the transactions.
fetchSet := make(map[chainhash.Hash]struct{})
// Fetch utxos for all of the transaction ouputs in this block.
// Typically, there will not be any utxos for any of the outputs.
fetchSet := make(map[wire.OutPoint]struct{})
for _, tx := range block.Transactions() {
fetchSet[*tx.Hash()] = struct{}{}
prevOut := wire.OutPoint{Hash: *tx.Hash()}
for txOutIdx := range tx.MsgTx().TxOut {
prevOut.Index = uint32(txOutIdx)
fetchSet[prevOut] = struct{}{}
}
}
err := view.fetchUtxos(b.db, fetchSet)
if err != nil {
@ -845,12 +847,12 @@ func (b *BlockChain) checkBIP0030(node *blockNode, block *btcutil.Block, view *U
// Duplicate transactions are only allowed if the previous transaction
// is fully spent.
for _, tx := range block.Transactions() {
txEntry := view.LookupEntry(tx.Hash())
if txEntry != nil && !txEntry.IsFullySpent() {
for outpoint := range fetchSet {
utxo := view.LookupEntry(outpoint)
if utxo != nil && !utxo.IsSpent() {
str := fmt.Sprintf("tried to overwrite transaction %v "+
"at block height %d that is not fully spent",
tx.Hash(), txEntry.blockHeight)
outpoint.Hash, utxo.BlockHeight())
return ruleError(ErrOverwriteTx, str)
}
}
@ -879,10 +881,8 @@ func CheckTransactionInputs(tx *btcutil.Tx, txHeight int32, utxoView *UtxoViewpo
var totalSatoshiIn int64
for txInIndex, txIn := range tx.MsgTx().TxIn {
// Ensure the referenced input transaction is available.
originTxHash := &txIn.PreviousOutPoint.Hash
originTxIndex := txIn.PreviousOutPoint.Index
utxoEntry := utxoView.LookupEntry(originTxHash)
if utxoEntry == nil || utxoEntry.IsOutputSpent(originTxIndex) {
utxo := utxoView.LookupEntry(txIn.PreviousOutPoint)
if utxo == nil || utxo.IsSpent() {
str := fmt.Sprintf("output %v referenced from "+
"transaction %s:%d either does not exist or "+
"has already been spent", txIn.PreviousOutPoint,
@ -892,15 +892,15 @@ func CheckTransactionInputs(tx *btcutil.Tx, txHeight int32, utxoView *UtxoViewpo
// Ensure the transaction is not spending coins which have not
// yet reached the required coinbase maturity.
if utxoEntry.IsCoinBase() {
originHeight := utxoEntry.BlockHeight()
if utxo.IsCoinBase() {
originHeight := utxo.BlockHeight()
blocksSincePrev := txHeight - originHeight
coinbaseMaturity := int32(chainParams.CoinbaseMaturity)
if blocksSincePrev < coinbaseMaturity {
str := fmt.Sprintf("tried to spend coinbase "+
"transaction %v from height %v at "+
"height %v before required maturity "+
"of %v blocks", originTxHash,
"transaction output %v from height %v "+
"at height %v before required maturity "+
"of %v blocks", txIn.PreviousOutPoint,
originHeight, txHeight,
coinbaseMaturity)
return 0, ruleError(ErrImmatureSpend, str)
@ -913,7 +913,7 @@ func CheckTransactionInputs(tx *btcutil.Tx, txHeight int32, utxoView *UtxoViewpo
// a transaction are in a unit value known as a satoshi. One
// bitcoin is a quantity of satoshi as defined by the
// SatoshiPerBitcoin constant.
originTxSatoshi := utxoEntry.AmountByIndex(originTxIndex)
originTxSatoshi := utxo.Amount()
if originTxSatoshi < 0 {
str := fmt.Sprintf("transaction output has negative "+
"value of %v", btcutil.Amount(originTxSatoshi))

26
blockchain/weight.go
View file

@ -1,4 +1,4 @@
// Copyright (c) 2013-2016 The btcsuite developers
// Copyright (c) 2013-2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
@ -8,6 +8,7 @@ import (
"fmt"
"github.com/btcsuite/btcd/txscript"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
)
@ -34,6 +35,14 @@ const (
// receives compared to "base" data. A scale factor of 4, denotes that
// witness data is 1/4 as cheap as regular non-witness data.
WitnessScaleFactor = 4
// MinTxOutputWeight is the minimum possible weight for a transaction
// output.
MinTxOutputWeight = WitnessScaleFactor * wire.MinTxOutPayload
// MaxOutputsPerBlock is the maximum number of transaction outputs there
// can be in a block of max weight size.
MaxOutputsPerBlock = MaxBlockWeight / MinTxOutputWeight
)
// GetBlockWeight computes the value of the weight metric for a given block.
@ -71,9 +80,7 @@ func GetTransactionWeight(tx *btcutil.Tx) int64 {
// legacy sig op count scaled according to the WitnessScaleFactor, the sig op
// count for all p2sh inputs scaled by the WitnessScaleFactor, and finally the
// unscaled sig op count for any inputs spending witness programs.
func GetSigOpCost(tx *btcutil.Tx, isCoinBaseTx bool, utxoView *UtxoViewpoint,
bip16, segWit bool) (int, error) {
func GetSigOpCost(tx *btcutil.Tx, isCoinBaseTx bool, utxoView *UtxoViewpoint, bip16, segWit bool) (int, error) {
numSigOps := CountSigOps(tx) * WitnessScaleFactor
if bip16 {
numP2SHSigOps, err := CountP2SHSigOps(tx, isCoinBaseTx, utxoView)
@ -86,11 +93,10 @@ func GetSigOpCost(tx *btcutil.Tx, isCoinBaseTx bool, utxoView *UtxoViewpoint,
if segWit && !isCoinBaseTx {
msgTx := tx.MsgTx()
for txInIndex, txIn := range msgTx.TxIn {
// Ensure the referenced input transaction is available.
originTxHash := &txIn.PreviousOutPoint.Hash
originTxIndex := txIn.PreviousOutPoint.Index
txEntry := utxoView.LookupEntry(originTxHash)
if txEntry == nil || txEntry.IsOutputSpent(originTxIndex) {
// Ensure the referenced output is available and hasn't
// already been spent.
utxo := utxoView.LookupEntry(txIn.PreviousOutPoint)
if utxo == nil || utxo.IsSpent() {
str := fmt.Sprintf("output %v referenced from "+
"transaction %s:%d either does not "+
"exist or has already been spent",
@ -101,7 +107,7 @@ func GetSigOpCost(tx *btcutil.Tx, isCoinBaseTx bool, utxoView *UtxoViewpoint,
witness := txIn.Witness
sigScript := txIn.SignatureScript
pkScript := txEntry.PkScriptByIndex(originTxIndex)
pkScript := utxo.PkScript()
numSigOps += txscript.GetWitnessSigOpCount(sigScript, pkScript, witness)
}

1
btcjson/chainsvrresults.go
View file

@ -289,7 +289,6 @@ type GetTxOutResult struct {
Confirmations int64 `json:"confirmations"`
Value float64 `json:"value"`
ScriptPubKey ScriptPubKeyResult `json:"scriptPubKey"`
Version int32 `json:"version"`
Coinbase bool `json:"coinbase"`
}

42
mempool/mempool.go
View file

@ -595,15 +595,21 @@ func (mp *TxPool) fetchInputUtxos(tx *btcutil.Tx) (*blockchain.UtxoViewpoint, er
}
// Attempt to populate any missing inputs from the transaction pool.
for originHash, entry := range utxoView.Entries() {
if entry != nil && !entry.IsFullySpent() {
for _, txIn := range tx.MsgTx().TxIn {
prevOut := &txIn.PreviousOutPoint
entry := utxoView.LookupEntry(*prevOut)
if entry != nil && !entry.IsSpent() {
continue
}
if poolTxDesc, exists := mp.pool[originHash]; exists {
utxoView.AddTxOuts(poolTxDesc.Tx, mining.UnminedHeight)
if poolTxDesc, exists := mp.pool[prevOut.Hash]; exists {
// AddTxOut ignores out of range index values, so it is
// safe to call without bounds checking here.
utxoView.AddTxOut(poolTxDesc.Tx, prevOut.Index,
mining.UnminedHeight)
}
}
return utxoView, nil
}
@ -733,25 +739,29 @@ func (mp *TxPool) maybeAcceptTransaction(tx *btcutil.Tx, isNew, rateLimit, rejec
// Don't allow the transaction if it exists in the main chain and is not
// not already fully spent.
txEntry := utxoView.LookupEntry(txHash)
if txEntry != nil && !txEntry.IsFullySpent() {
return nil, nil, txRuleError(wire.RejectDuplicate,
"transaction already exists")
prevOut := wire.OutPoint{Hash: *txHash}
for txOutIdx := range tx.MsgTx().TxOut {
prevOut.Index = uint32(txOutIdx)
entry := utxoView.LookupEntry(prevOut)
if entry != nil && !entry.IsSpent() {
return nil, nil, txRuleError(wire.RejectDuplicate,
"transaction already exists")
}
utxoView.RemoveEntry(prevOut)
}
delete(utxoView.Entries(), *txHash)
// Transaction is an orphan if any of the referenced input transactions
// don't exist. Adding orphans to the orphan pool is not handled by
// this function, and the caller should use maybeAddOrphan if this
// behavior is desired.
// Transaction is an orphan if any of the referenced transaction outputs
// don't exist or are already spent. Adding orphans to the orphan pool
// is not handled by this function, and the caller should use
// maybeAddOrphan if this behavior is desired.
var missingParents []*chainhash.Hash
for originHash, entry := range utxoView.Entries() {
if entry == nil || entry.IsFullySpent() {
for outpoint, entry := range utxoView.Entries() {
if entry == nil || entry.IsSpent() {
// Must make a copy of the hash here since the iterator
// is replaced and taking its address directly would
// result in all of the entries pointing to the same
// memory location and thus all be the final hash.
hashCopy := originHash
hashCopy := outpoint.Hash
missingParents = append(missingParents, &hashCopy)
}
}

21
mempool/mempool_test.go
View file

@ -31,10 +31,10 @@ type fakeChain struct {
medianTimePast time.Time
}
// FetchUtxoView loads utxo details about the input transactions referenced by
// the passed transaction from the point of view of the fake chain.
// It also attempts to fetch the utxo details for the transaction itself so the
// returned view can be examined for duplicate unspent transaction outputs.
// FetchUtxoView loads utxo details about the inputs referenced by the passed
// transaction from the point of view of the fake chain. It also attempts to
// fetch the utxos for the outputs of the transaction itself so the returned
// view can be examined for duplicate transactions.
//
// This function is safe for concurrent access however the returned view is NOT.
func (s *fakeChain) FetchUtxoView(tx *btcutil.Tx) (*blockchain.UtxoViewpoint, error) {
@ -46,14 +46,17 @@ func (s *fakeChain) FetchUtxoView(tx *btcutil.Tx) (*blockchain.UtxoViewpoint, er
// Add an entry for the tx itself to the new view.
viewpoint := blockchain.NewUtxoViewpoint()
entry := s.utxos.LookupEntry(tx.Hash())
viewpoint.Entries()[*tx.Hash()] = entry.Clone()
prevOut := wire.OutPoint{Hash: *tx.Hash()}
for txOutIdx := range tx.MsgTx().TxOut {
prevOut.Index = uint32(txOutIdx)
entry := s.utxos.LookupEntry(prevOut)
viewpoint.Entries()[prevOut] = entry.Clone()
}
// Add entries for all of the inputs to the tx to the new view.
for _, txIn := range tx.MsgTx().TxIn {
originHash := &txIn.PreviousOutPoint.Hash
entry := s.utxos.LookupEntry(originHash)
viewpoint.Entries()[*originHash] = entry.Clone()
entry := s.utxos.LookupEntry(txIn.PreviousOutPoint)
viewpoint.Entries()[txIn.PreviousOutPoint] = entry.Clone()
}
return viewpoint, nil

5
mempool/policy.go
View file

@ -98,9 +98,8 @@ func checkInputsStandard(tx *btcutil.Tx, utxoView *blockchain.UtxoViewpoint) err
// It is safe to elide existence and index checks here since
// they have already been checked prior to calling this
// function.
prevOut := txIn.PreviousOutPoint
entry := utxoView.LookupEntry(&prevOut.Hash)
originPkScript := entry.PkScriptByIndex(prevOut.Index)
entry := utxoView.LookupEntry(txIn.PreviousOutPoint)
originPkScript := entry.PkScript()
switch txscript.GetScriptClass(originPkScript) {
case txscript.ScriptHashTy:
numSigOps := txscript.GetPreciseSigOpCount(

21
mining/mining.go
View file

@ -222,14 +222,14 @@ type BlockTemplate struct {
// mergeUtxoView adds all of the entries in viewB to viewA. The result is that
// viewA will contain all of its original entries plus all of the entries
// in viewB. It will replace any entries in viewB which also exist in viewA
// if the entry in viewA is fully spent.
// if the entry in viewA is spent.
func mergeUtxoView(viewA *blockchain.UtxoViewpoint, viewB *blockchain.UtxoViewpoint) {
viewAEntries := viewA.Entries()
for hash, entryB := range viewB.Entries() {
if entryA, exists := viewAEntries[hash]; !exists ||
entryA == nil || entryA.IsFullySpent() {
for outpoint, entryB := range viewB.Entries() {
if entryA, exists := viewAEntries[outpoint]; !exists ||
entryA == nil || entryA.IsSpent() {
viewAEntries[hash] = entryB
viewAEntries[outpoint] = entryB
}
}
}
@ -291,11 +291,9 @@ func createCoinbaseTx(params *chaincfg.Params, coinbaseScript []byte, nextBlockH
// which are not provably unspendable as available unspent transaction outputs.
func spendTransaction(utxoView *blockchain.UtxoViewpoint, tx *btcutil.Tx, height int32) error {
for _, txIn := range tx.MsgTx().TxIn {
originHash := &txIn.PreviousOutPoint.Hash
originIndex := txIn.PreviousOutPoint.Index
entry := utxoView.LookupEntry(originHash)
entry := utxoView.LookupEntry(txIn.PreviousOutPoint)
if entry != nil {
entry.SpendOutput(originIndex)
entry.Spend()
}
}
@ -540,9 +538,8 @@ mempoolLoop:
prioItem := &txPrioItem{tx: tx}
for _, txIn := range tx.MsgTx().TxIn {
originHash := &txIn.PreviousOutPoint.Hash
originIndex := txIn.PreviousOutPoint.Index
utxoEntry := utxos.LookupEntry(originHash)
if utxoEntry == nil || utxoEntry.IsOutputSpent(originIndex) {
entry := utxos.LookupEntry(txIn.PreviousOutPoint)
if entry == nil || entry.IsSpent() {
if !g.txSource.HaveTransaction(originHash) {
log.Tracef("Skipping tx %s because it "+
"references unspent output %s "+

10
mining/policy.go
View file

@ -67,16 +67,14 @@ func calcInputValueAge(tx *wire.MsgTx, utxoView *blockchain.UtxoViewpoint, nextB
for _, txIn := range tx.TxIn {
// Don't attempt to accumulate the total input age if the
// referenced transaction output doesn't exist.
originHash := &txIn.PreviousOutPoint.Hash
originIndex := txIn.PreviousOutPoint.Index
txEntry := utxoView.LookupEntry(originHash)
if txEntry != nil && !txEntry.IsOutputSpent(originIndex) {
entry := utxoView.LookupEntry(txIn.PreviousOutPoint)
if entry != nil && !entry.IsSpent() {
// Inputs with dependencies currently in the mempool
// have their block height set to a special constant.
// Their input age should computed as zero since their
// parent hasn't made it into a block yet.
var inputAge int32
originHeight := txEntry.BlockHeight()
originHeight := entry.BlockHeight()
if originHeight == UnminedHeight {
inputAge = 0
} else {
@ -84,7 +82,7 @@ func calcInputValueAge(tx *wire.MsgTx, utxoView *blockchain.UtxoViewpoint, nextB
}
// Sum the input value times age.
inputValue := txEntry.AmountByIndex(originIndex)
inputValue := entry.Amount()
totalInputAge += float64(inputValue * int64(inputAge))
}
}

24
netsync/manager.go
View file

@ -898,12 +898,26 @@ func (sm *SyncManager) haveInventory(invVect *wire.InvVect) (bool, error) {
}
// Check if the transaction exists from the point of view of the
// end of the main chain.
entry, err := sm.chain.FetchUtxoEntry(&invVect.Hash)
if err != nil {
return false, err
// end of the main chain. Note that this is only a best effort
// since it is expensive to check existence of every output and
// the only purpose of this check is to avoid downloading
// already known transactions. Only the first two outputs are
// checked because the vast majority of transactions consist of
// two outputs where one is some form of "pay-to-somebody-else"
// and the other is a change output.
prevOut := wire.OutPoint{Hash: invVect.Hash}
for i := uint32(0); i < 2; i++ {
prevOut.Index = i
entry, err := sm.chain.FetchUtxoEntry(prevOut)
if err != nil {
return false, err
}
if entry != nil && !entry.IsSpent() {
return true, nil
}
}
return entry != nil && !entry.IsFullySpent(), nil
return false, nil
}
// The requested inventory is is an unsupported type, so just claim

13
rpcserver.go
View file

@ -2667,7 +2667,6 @@ func handleGetTxOut(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (i
// from there, otherwise attempt to fetch from the block database.
var bestBlockHash string
var confirmations int32
var txVersion int32
var value int64
var pkScript []byte
var isCoinbase bool
@ -2702,12 +2701,12 @@ func handleGetTxOut(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (i
best := s.cfg.Chain.BestSnapshot()
bestBlockHash = best.Hash.String()
confirmations = 0
txVersion = mtx.Version
value = txOut.Value
pkScript = txOut.PkScript
isCoinbase = blockchain.IsCoinBaseTx(mtx)
} else {
entry, err := s.cfg.Chain.FetchUtxoEntry(txHash)
out := wire.OutPoint{Hash: *txHash, Index: c.Vout}
entry, err := s.cfg.Chain.FetchUtxoEntry(out)
if err != nil {
return nil, rpcNoTxInfoError(txHash)
}
@ -2717,16 +2716,15 @@ func handleGetTxOut(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (i
// transaction already in the main chain. Mined transactions
// that are spent by a mempool transaction are not affected by
// this.
if entry == nil || entry.IsOutputSpent(c.Vout) {
if entry == nil || entry.IsSpent() {
return nil, nil
}
best := s.cfg.Chain.BestSnapshot()
bestBlockHash = best.Hash.String()
confirmations = 1 + best.Height - entry.BlockHeight()
txVersion = entry.Version()
value = entry.AmountByIndex(c.Vout)
pkScript = entry.PkScriptByIndex(c.Vout)
value = entry.Amount()
pkScript = entry.PkScript()
isCoinbase = entry.IsCoinBase()
}
@ -2749,7 +2747,6 @@ func handleGetTxOut(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (i
BestBlock: bestBlockHash,
Confirmations: int64(confirmations),
Value: btcutil.Amount(value).ToBTC(),
Version: txVersion,
ScriptPubKey: btcjson.ScriptPubKeyResult{
Asm: disbuf,
Hex: hex.EncodeToString(pkScript),

6
wire/msgtx.go
View file

@ -62,13 +62,13 @@ const (
// a transaction which fits into a message could possibly have.
maxTxInPerMessage = (MaxMessagePayload / minTxInPayload) + 1
// minTxOutPayload is the minimum payload size for a transaction output.
// MinTxOutPayload is the minimum payload size for a transaction output.
// Value 8 bytes + Varint for PkScript length 1 byte.
minTxOutPayload = 9
MinTxOutPayload = 9
// maxTxOutPerMessage is the maximum number of transactions outputs that
// a transaction which fits into a message could possibly have.
maxTxOutPerMessage = (MaxMessagePayload / minTxOutPayload) + 1
maxTxOutPerMessage = (MaxMessagePayload / MinTxOutPayload) + 1
// minTxPayload is the minimum payload size for a transaction. Note
// that any realistically usable transaction must have at least one