Optimize transaction lookups.
This commit modifies the transaction lookup code to use a set instead of a slice (list). This allows the lookup to automatically prevent duplicate requests to the database. Previously, the code simply added every referenced transaction to a list without checking for duplicates, which led to multiple requests against the database for the same transaction. It also meant the request list could grow quite large with all of the duplicates using far more memory than required. While the end result was accurate, operating that way is not as efficient as only requesting unique transactions.
This commit is contained in:
Dave Collins
parent
32790d52d8
commit
e888372019
4 changed files with 35 additions and 52 deletions
|
|
@ -16,7 +16,6 @@ func TestHaveBlock(t *testing.T) {
|
|||
// Load up blocks such that there is a side chain.
|
||||
// (genesis block) -> 1 -> 2 -> 3 -> 4
|
||||
// \-> 3a
|
||||
// orphans are handled properly along with chain reorganization.
|
||||
testFiles := []string{
|
||||
"blk_0_to_4.dat.bz2",
|
||||
"blk_3A.dat.bz2",
|
||||
|
|
|
|||
|
|
@ -20,14 +20,14 @@ github.com/conformal/btcchain/timesorter.go timeSorter.Swap 100.00% (1/1)
|
|||
github.com/conformal/btcchain/checkpoints.go BlockChain.DisableCheckpoints 100.00% (1/1)
|
||||
github.com/conformal/btcchain/params.go BlockChain.chainParams 100.00% (1/1)
|
||||
github.com/conformal/btcchain/log.go init 100.00% (1/1)
|
||||
github.com/conformal/btcchain/txlookup.go fetchTxStoreMain 94.44% (17/18)
|
||||
github.com/conformal/btcchain/merkle.go BuildMerkleTreeStore 94.12% (16/17)
|
||||
github.com/conformal/btcchain/txlookup.go disconnectTransactions 93.75% (15/16)
|
||||
github.com/conformal/btcchain/txlookup.go fetchTxListMain 93.33% (14/15)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.getReorganizeNodes 92.86% (13/14)
|
||||
github.com/conformal/btcchain/process.go BlockChain.processOrphans 92.86% (13/14)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.connectBestChain 90.00% (27/30)
|
||||
github.com/conformal/btcchain/scriptval.go ValidateTransactionScripts 88.24% (30/34)
|
||||
github.com/conformal/btcchain/txlookup.go BlockChain.fetchTxList 86.36% (19/22)
|
||||
github.com/conformal/btcchain/txlookup.go BlockChain.fetchTxStore 86.36% (19/22)
|
||||
github.com/conformal/btcchain/scriptval.go checkBlockScripts 85.71% (6/7)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.reorganizeChain 85.29% (29/34)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.connectBlock 83.33% (10/12)
|
||||
|
|
@ -37,13 +37,14 @@ github.com/conformal/btcchain/chain.go BlockChain.isMajorityVersion 80.00% (
|
|||
github.com/conformal/btcchain/difficulty.go calcWork 80.00% (4/5)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.addOrphanBlock 77.78% (14/18)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.getPrevNodeFromBlock 77.78% (7/9)
|
||||
github.com/conformal/btcchain/txlookup.go BlockChain.fetchInputTransactions 76.92% (20/26)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.disconnectBlock 76.92% (10/13)
|
||||
github.com/conformal/btcchain/txlookup.go BlockChain.fetchInputTransactions 76.00% (19/25)
|
||||
github.com/conformal/btcchain/difficulty.go BigToCompact 75.00% (12/16)
|
||||
github.com/conformal/btcchain/difficulty.go CompactToBig 75.00% (9/12)
|
||||
github.com/conformal/btcchain/validate.go BlockChain.checkConnectBlock 69.23% (36/52)
|
||||
github.com/conformal/btcchain/validate.go isNullOutpoint 66.67% (2/3)
|
||||
github.com/conformal/btcchain/validate.go BlockChain.checkBlockSanity 65.12% (28/43)
|
||||
github.com/conformal/btcchain/validate.go BlockChain.checkBIP0030 64.71% (11/17)
|
||||
github.com/conformal/btcchain/scriptval.go validateTxIn 64.71% (11/17)
|
||||
github.com/conformal/btcchain/validate.go CheckTransactionInputs 64.44% (29/45)
|
||||
github.com/conformal/btcchain/validate.go CheckTransactionSanity 62.16% (23/37)
|
||||
|
|
@ -51,7 +52,6 @@ github.com/conformal/btcchain/txlookup.go connectTransactions 60.00% (9/15)
|
|||
github.com/conformal/btcchain/params.go ChainParams 60.00% (3/5)
|
||||
github.com/conformal/btcchain/validate.go isBIP0030Node 60.00% (3/5)
|
||||
github.com/conformal/btcchain/validate.go BlockChain.checkProofOfWork 58.82% (10/17)
|
||||
github.com/conformal/btcchain/validate.go BlockChain.checkBIP0030 57.14% (8/14)
|
||||
github.com/conformal/btcchain/process.go BlockChain.ProcessBlock 53.49% (23/43)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.loadBlockNode 50.00% (11/22)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.getPrevNodeFromNode 50.00% (4/8)
|
||||
|
|
@ -68,10 +68,10 @@ github.com/conformal/btcchain/checkpoints.go BlockChain.IsCheckpointCandidate
|
|||
github.com/conformal/btcchain/validate.go countP2SHSigOps 0.00% (0/24)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.GenerateInitialIndex 0.00% (0/17)
|
||||
github.com/conformal/btcchain/difficulty.go BlockChain.calcEasiestDifficulty 0.00% (0/15)
|
||||
github.com/conformal/btcchain/txlookup.go BlockChain.FetchTransactionStore 0.00% (0/15)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.removeBlockNode 0.00% (0/12)
|
||||
github.com/conformal/btcchain/difficulty.go BlockChain.findPrevTestNetDifficulty 0.00% (0/12)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.GetOrphanRoot 0.00% (0/11)
|
||||
github.com/conformal/btcchain/txlookup.go BlockChain.FetchTransactionStore 0.00% (0/9)
|
||||
github.com/conformal/btcchain/chain.go BlockChain.IsCurrent 0.00% (0/9)
|
||||
github.com/conformal/btcchain/chain.go removeChildNode 0.00% (0/8)
|
||||
github.com/conformal/btcchain/log.go SetLogWriter 0.00% (0/7)
|
||||
|
|
@ -86,5 +86,5 @@ github.com/conformal/btcchain/log.go newLogClosure 0.00% (0/1)
|
|||
github.com/conformal/btcchain/chain.go BlockChain.DisableVerify 0.00% (0/1)
|
||||
github.com/conformal/btcchain/log.go UseLogger 0.00% (0/1)
|
||||
github.com/conformal/btcchain/process.go RuleError.Error 0.00% (0/1)
|
||||
github.com/conformal/btcchain ------------------------------------- 55.40% (641/1157)
|
||||
github.com/conformal/btcchain ------------------------------------- 55.88% (646/1156)
|
||||
|
||||
|
|
|
|||
68
txlookup.go
68
txlookup.go
|
|
@ -99,15 +99,18 @@ func disconnectTransactions(txStore TxStore, block *btcutil.Block) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// fetchTxListMain fetches transaction data about the provided list of
|
||||
// fetchTxStoreMain fetches transaction data about the provided set of
|
||||
// transactions from the point of view of the end of the main chain.
|
||||
func fetchTxListMain(db btcdb.Db, txList []*btcwire.ShaHash) TxStore {
|
||||
func fetchTxStoreMain(db btcdb.Db, txSet map[btcwire.ShaHash]bool) TxStore {
|
||||
// The transaction store map needs to have an entry for every requested
|
||||
// transaction. By default, all the transactions are marked as missing.
|
||||
// Each entry will be filled in with the appropriate data below.
|
||||
txList := make([]*btcwire.ShaHash, 0, len(txSet))
|
||||
txStore := make(TxStore)
|
||||
for _, hash := range txList {
|
||||
txStore[*hash] = &TxData{Hash: hash, Err: btcdb.TxShaMissing}
|
||||
for hash := range txSet {
|
||||
hashCopy := hash
|
||||
txStore[hash] = &TxData{Hash: &hashCopy, Err: btcdb.TxShaMissing}
|
||||
txList = append(txList, &hashCopy)
|
||||
}
|
||||
|
||||
// Ask the database (main chain) for the list of transactions. This
|
||||
|
|
@ -141,14 +144,14 @@ func fetchTxListMain(db btcdb.Db, txList []*btcwire.ShaHash) TxStore {
|
|||
return txStore
|
||||
}
|
||||
|
||||
// fetchTxList fetches transaction data about the provided list of transactions
|
||||
// fetchTxStore fetches transaction data about the provided set of transactions
|
||||
// from the point of view of the given node. For example, a given node might
|
||||
// be down a side chain where a transaction hasn't been spent from its point of
|
||||
// view even though it might have been spent in the main chain (or another side
|
||||
// chain). Another scenario is where a transaction exists from the point of
|
||||
// view of the main chain, but doesn't exist in a side chain that branches
|
||||
// before the block that contains the transaction on the main chain.
|
||||
func (b *BlockChain) fetchTxList(node *blockNode, txList []*btcwire.ShaHash) (TxStore, error) {
|
||||
func (b *BlockChain) fetchTxStore(node *blockNode, txSet map[btcwire.ShaHash]bool) (TxStore, error) {
|
||||
// Get the previous block node. This function is used over simply
|
||||
// accessing node.parent directly as it will dynamically create previous
|
||||
// block nodes as needed. This helps allow only the pieces of the chain
|
||||
|
|
@ -158,9 +161,9 @@ func (b *BlockChain) fetchTxList(node *blockNode, txList []*btcwire.ShaHash) (Tx
|
|||
return nil, err
|
||||
}
|
||||
|
||||
// Fetch the requested list from the point of view of the end of the
|
||||
// Fetch the requested set from the point of view of the end of the
|
||||
// main (best) chain.
|
||||
txStore := fetchTxListMain(b.db, txList)
|
||||
txStore := fetchTxStoreMain(b.db, txSet)
|
||||
|
||||
// If we haven't selected a best chain yet or we are extending the main
|
||||
// (best) chain with a new block, everything is accurate, so return the
|
||||
|
|
@ -230,20 +233,10 @@ func (b *BlockChain) fetchInputTransactions(node *blockNode, block *btcutil.Bloc
|
|||
txInFlight[*txHash] = i
|
||||
}
|
||||
|
||||
// Make a reasonable guess for the maximum number of needed input
|
||||
// transactions to use as the starting point for the needed transactions
|
||||
// array. The array will dynamically grow as needed, but it's much less
|
||||
// overhead to avoid growing and copying the array multiple times in the
|
||||
// common case. Each block usually has no more than ten inputs per
|
||||
// transaction, so use that as a reasonable starting point. A block
|
||||
// with 2,000 transactions would only result in around 156KB on a 64-bit
|
||||
// system using this approach.
|
||||
maxNeededHint := (len(transactions) - 1) * 10
|
||||
txNeededList := make([]*btcwire.ShaHash, 0, maxNeededHint)
|
||||
|
||||
// Loop through all of the transaction inputs (except for the coinbase
|
||||
// which has no inputs) collecting them into lists of what is needed and
|
||||
// which has no inputs) collecting them into sets of what is needed and
|
||||
// what is already known (in-flight).
|
||||
txNeededSet := make(map[btcwire.ShaHash]bool)
|
||||
txStore := make(TxStore)
|
||||
for i, tx := range transactions[1:] {
|
||||
for _, txIn := range tx.TxIn {
|
||||
|
|
@ -272,13 +265,13 @@ func (b *BlockChain) fetchInputTransactions(node *blockNode, block *btcutil.Bloc
|
|||
txD.Spent = make([]bool, len(originTx.TxOut))
|
||||
txD.Err = nil
|
||||
} else {
|
||||
txNeededList = append(txNeededList, originHash)
|
||||
txNeededSet[*originHash] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Request the input transactions from the point of view of the node.
|
||||
txNeededStore, err := b.fetchTxList(node, txNeededList)
|
||||
txNeededStore, err := b.fetchTxStore(node, txNeededSet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -302,30 +295,17 @@ func (b *BlockChain) FetchTransactionStore(tx *btcwire.MsgTx) (TxStore, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
// Create list big
|
||||
txNeededList := make([]*btcwire.ShaHash, 0, len(tx.TxIn)+1)
|
||||
txNeededList = append(txNeededList, &txHash)
|
||||
|
||||
// Loop through all of the transaction inputs collecting them into lists of what is needed and
|
||||
// what is already known (in-flight).
|
||||
txStore := make(TxStore)
|
||||
// Create a set of needed transactions from the transactions referenced
|
||||
// by the inputs of the passed transaction. Also, add the passed
|
||||
// transaction itself as a way for the caller to detect duplicates.
|
||||
txNeededSet := make(map[btcwire.ShaHash]bool)
|
||||
txNeededSet[txHash] = true
|
||||
for _, txIn := range tx.TxIn {
|
||||
// Add an entry to the transaction store for the needed
|
||||
// transaction with it set to missing by default.
|
||||
originHash := &txIn.PreviousOutpoint.Hash
|
||||
txD := &TxData{Hash: originHash, Err: btcdb.TxShaMissing}
|
||||
txStore[*originHash] = txD
|
||||
txNeededList = append(txNeededList, originHash)
|
||||
}
|
||||
|
||||
// Request the input transactions from the point of view of the node.
|
||||
txNeededStore := fetchTxListMain(b.db, txNeededList)
|
||||
|
||||
// Merge the results of the requested transactions and the in-flight
|
||||
// transactions.
|
||||
for _, txD := range txNeededStore {
|
||||
txStore[*txD.Hash] = txD
|
||||
txNeededSet[txIn.PreviousOutpoint.Hash] = true
|
||||
}
|
||||
|
||||
// Request the input transactions from the point of view of the end of
|
||||
// the main chain.
|
||||
txStore := fetchTxStoreMain(b.db, txNeededSet)
|
||||
return txStore, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -555,7 +555,11 @@ func (b *BlockChain) checkBIP0030(node *blockNode, block *btcutil.Block) error {
|
|||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
txResults, err := b.fetchTxList(node, fetchList)
|
||||
fetchSet := make(map[btcwire.ShaHash]bool)
|
||||
for _, txHash := range fetchList {
|
||||
fetchSet[*txHash] = true
|
||||
}
|
||||
txResults, err := b.fetchTxStore(node, fetchSet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue