Optimize transaction lookups.

This commit modifies the transaction lookup code to use a set instead of a
slice (list).  This allows the lookup to automatically prevent duplicate
requests to the database.

Previously, the code simply added every referenced transaction to a list
without checking for duplicates, which led to multiple requests against
the database for the same transaction.  It also meant the request list
could grow quite large with all of the duplicates using far more memory
than required.

While the end result was accurate, operating that way is not as efficient
as only requesting unique transactions.

chain_test.go

@@ -16,7 +16,6 @@ func TestHaveBlock(t *testing.T) {
 	// Load up blocks such that there is a side chain.
 	// (genesis block) -> 1 -> 2 -> 3 -> 4
 	//                          \-> 3a
-	// orphans are handled properly along with chain reorganization.
 	testFiles := []string{
 		"blk_0_to_4.dat.bz2",
 		"blk_3A.dat.bz2",

test_coverage.txt

@@ -20,14 +20,14 @@ github.com/conformal/btcchain/timesorter.go	 timeSorter.Swap			 100.00% (1/1)
 github.com/conformal/btcchain/checkpoints.go	 BlockChain.DisableCheckpoints		 100.00% (1/1)
 github.com/conformal/btcchain/params.go		 BlockChain.chainParams			 100.00% (1/1)
 github.com/conformal/btcchain/log.go		 init					 100.00% (1/1)
+github.com/conformal/btcchain/txlookup.go	 fetchTxStoreMain			 94.44% (17/18)
 github.com/conformal/btcchain/merkle.go		 BuildMerkleTreeStore			 94.12% (16/17)
 github.com/conformal/btcchain/txlookup.go	 disconnectTransactions			 93.75% (15/16)
-github.com/conformal/btcchain/txlookup.go	 fetchTxListMain			 93.33% (14/15)
 github.com/conformal/btcchain/chain.go		 BlockChain.getReorganizeNodes		 92.86% (13/14)
 github.com/conformal/btcchain/process.go	 BlockChain.processOrphans		 92.86% (13/14)
 github.com/conformal/btcchain/chain.go		 BlockChain.connectBestChain		 90.00% (27/30)
 github.com/conformal/btcchain/scriptval.go	 ValidateTransactionScripts		 88.24% (30/34)
-github.com/conformal/btcchain/txlookup.go	 BlockChain.fetchTxList			 86.36% (19/22)
+github.com/conformal/btcchain/txlookup.go	 BlockChain.fetchTxStore		 86.36% (19/22)
 github.com/conformal/btcchain/scriptval.go	 checkBlockScripts			 85.71% (6/7)
 github.com/conformal/btcchain/chain.go		 BlockChain.reorganizeChain		 85.29% (29/34)
 github.com/conformal/btcchain/chain.go		 BlockChain.connectBlock		 83.33% (10/12)
@@ -37,13 +37,14 @@ github.com/conformal/btcchain/chain.go		 BlockChain.isMajorityVersion		 80.00% (
 github.com/conformal/btcchain/difficulty.go	 calcWork				 80.00% (4/5)
 github.com/conformal/btcchain/chain.go		 BlockChain.addOrphanBlock		 77.78% (14/18)
 github.com/conformal/btcchain/chain.go		 BlockChain.getPrevNodeFromBlock	 77.78% (7/9)
-github.com/conformal/btcchain/txlookup.go	 BlockChain.fetchInputTransactions	 76.92% (20/26)
 github.com/conformal/btcchain/chain.go		 BlockChain.disconnectBlock		 76.92% (10/13)
+github.com/conformal/btcchain/txlookup.go	 BlockChain.fetchInputTransactions	 76.00% (19/25)
 github.com/conformal/btcchain/difficulty.go	 BigToCompact				 75.00% (12/16)
 github.com/conformal/btcchain/difficulty.go	 CompactToBig				 75.00% (9/12)
 github.com/conformal/btcchain/validate.go	 BlockChain.checkConnectBlock		 69.23% (36/52)
 github.com/conformal/btcchain/validate.go	 isNullOutpoint				 66.67% (2/3)
 github.com/conformal/btcchain/validate.go	 BlockChain.checkBlockSanity		 65.12% (28/43)
+github.com/conformal/btcchain/validate.go	 BlockChain.checkBIP0030		 64.71% (11/17)
 github.com/conformal/btcchain/scriptval.go	 validateTxIn				 64.71% (11/17)
 github.com/conformal/btcchain/validate.go	 CheckTransactionInputs			 64.44% (29/45)
 github.com/conformal/btcchain/validate.go	 CheckTransactionSanity			 62.16% (23/37)
@@ -51,7 +52,6 @@ github.com/conformal/btcchain/txlookup.go	 connectTransactions			 60.00% (9/15)
 github.com/conformal/btcchain/params.go		 ChainParams				 60.00% (3/5)
 github.com/conformal/btcchain/validate.go	 isBIP0030Node				 60.00% (3/5)
 github.com/conformal/btcchain/validate.go	 BlockChain.checkProofOfWork		 58.82% (10/17)
-github.com/conformal/btcchain/validate.go	 BlockChain.checkBIP0030		 57.14% (8/14)
 github.com/conformal/btcchain/process.go	 BlockChain.ProcessBlock		 53.49% (23/43)
 github.com/conformal/btcchain/chain.go		 BlockChain.loadBlockNode		 50.00% (11/22)
 github.com/conformal/btcchain/chain.go		 BlockChain.getPrevNodeFromNode		 50.00% (4/8)
@@ -68,10 +68,10 @@ github.com/conformal/btcchain/checkpoints.go	 BlockChain.IsCheckpointCandidate
 github.com/conformal/btcchain/validate.go	 countP2SHSigOps			 0.00% (0/24)
 github.com/conformal/btcchain/chain.go		 BlockChain.GenerateInitialIndex	 0.00% (0/17)
 github.com/conformal/btcchain/difficulty.go	 BlockChain.calcEasiestDifficulty	 0.00% (0/15)
-github.com/conformal/btcchain/txlookup.go	 BlockChain.FetchTransactionStore	 0.00% (0/15)
 github.com/conformal/btcchain/chain.go		 BlockChain.removeBlockNode		 0.00% (0/12)
 github.com/conformal/btcchain/difficulty.go	 BlockChain.findPrevTestNetDifficulty	 0.00% (0/12)
 github.com/conformal/btcchain/chain.go		 BlockChain.GetOrphanRoot		 0.00% (0/11)
+github.com/conformal/btcchain/txlookup.go	 BlockChain.FetchTransactionStore	 0.00% (0/9)
 github.com/conformal/btcchain/chain.go		 BlockChain.IsCurrent			 0.00% (0/9)
 github.com/conformal/btcchain/chain.go		 removeChildNode			 0.00% (0/8)
 github.com/conformal/btcchain/log.go		 SetLogWriter				 0.00% (0/7)
@@ -86,5 +86,5 @@ github.com/conformal/btcchain/log.go		 newLogClosure				 0.00% (0/1)
 github.com/conformal/btcchain/chain.go		 BlockChain.DisableVerify		 0.00% (0/1)
 github.com/conformal/btcchain/log.go		 UseLogger				 0.00% (0/1)
 github.com/conformal/btcchain/process.go	 RuleError.Error			 0.00% (0/1)
-github.com/conformal/btcchain			 -------------------------------------	 55.40% (641/1157)
+github.com/conformal/btcchain			 -------------------------------------	 55.88% (646/1156)
 

txlookup.go

@@ -99,15 +99,18 @@ func disconnectTransactions(txStore TxStore, block *btcutil.Block) error {
 	return nil
 }
 
-// fetchTxListMain fetches transaction data about the provided list of
+// fetchTxStoreMain fetches transaction data about the provided set of
 // transactions from the point of view of the end of the main chain.
-func fetchTxListMain(db btcdb.Db, txList []*btcwire.ShaHash) TxStore {
+func fetchTxStoreMain(db btcdb.Db, txSet map[btcwire.ShaHash]bool) TxStore {
 	// The transaction store map needs to have an entry for every requested
 	// transaction.  By default, all the transactions are marked as missing.
 	// Each entry will be filled in with the appropriate data below.
+	txList := make([]*btcwire.ShaHash, 0, len(txSet))
 	txStore := make(TxStore)
-	for _, hash := range txList {
+	for hash := range txSet {
-		txStore[*hash] = &TxData{Hash: hash, Err: btcdb.TxShaMissing}
+		hashCopy := hash
+		txStore[hash] = &TxData{Hash: &hashCopy, Err: btcdb.TxShaMissing}
+		txList = append(txList, &hashCopy)
 	}
 
 	// Ask the database (main chain) for the list of transactions.  This
@@ -141,14 +144,14 @@ func fetchTxListMain(db btcdb.Db, txList []*btcwire.ShaHash) TxStore {
 	return txStore
 }
 
-// fetchTxList fetches transaction data about the provided list of transactions
+// fetchTxStore fetches transaction data about the provided set of transactions
 // from the point of view of the given node.  For example, a given node might
 // be down a side chain where a transaction hasn't been spent from its point of
 // view even though it might have been spent in the main chain (or another side
 // chain).  Another scenario is where a transaction exists from the point of
 // view of the main chain, but doesn't exist in a side chain that branches
 // before the block that contains the transaction on the main chain.
-func (b *BlockChain) fetchTxList(node *blockNode, txList []*btcwire.ShaHash) (TxStore, error) {
+func (b *BlockChain) fetchTxStore(node *blockNode, txSet map[btcwire.ShaHash]bool) (TxStore, error) {
 	// Get the previous block node.  This function is used over simply
 	// accessing node.parent directly as it will dynamically create previous
 	// block nodes as needed.  This helps allow only the pieces of the chain
@@ -158,9 +161,9 @@ func (b *BlockChain) fetchTxList(node *blockNode, txList []*btcwire.ShaHash) (Tx
 		return nil, err
 	}
 
-	// Fetch the requested list from the point of view of the end of the
+	// Fetch the requested set from the point of view of the end of the
 	// main (best) chain.
-	txStore := fetchTxListMain(b.db, txList)
+	txStore := fetchTxStoreMain(b.db, txSet)
 
 	// If we haven't selected a best chain yet or we are extending the main
 	// (best) chain with a new block, everything is accurate, so return the
@@ -230,20 +233,10 @@ func (b *BlockChain) fetchInputTransactions(node *blockNode, block *btcutil.Bloc
 		txInFlight[*txHash] = i
 	}
 
-	// Make a reasonable guess for the maximum number of needed input
-	// transactions to use as the starting point for the needed transactions
-	// array.  The array will dynamically grow as needed, but it's much less
-	// overhead to avoid growing and copying the array multiple times in the
-	// common case.  Each block usually has no more than ten inputs per
-	// transaction, so use that as a reasonable starting point.  A block
-	// with 2,000 transactions would only result in around 156KB on a 64-bit
-	// system using this approach.
-	maxNeededHint := (len(transactions) - 1) * 10
-	txNeededList := make([]*btcwire.ShaHash, 0, maxNeededHint)
-
 	// Loop through all of the transaction inputs (except for the coinbase
-	// which has no inputs) collecting them into lists of what is needed and
+	// which has no inputs) collecting them into sets of what is needed and
 	// what is already known (in-flight).
+	txNeededSet := make(map[btcwire.ShaHash]bool)
 	txStore := make(TxStore)
 	for i, tx := range transactions[1:] {
 		for _, txIn := range tx.TxIn {
@@ -272,13 +265,13 @@ func (b *BlockChain) fetchInputTransactions(node *blockNode, block *btcutil.Bloc
 				txD.Spent = make([]bool, len(originTx.TxOut))
 				txD.Err = nil
 			} else {
-				txNeededList = append(txNeededList, originHash)
+				txNeededSet[*originHash] = true
 			}
 		}
 	}
 
 	// Request the input transactions from the point of view of the node.
-	txNeededStore, err := b.fetchTxList(node, txNeededList)
+	txNeededStore, err := b.fetchTxStore(node, txNeededSet)
 	if err != nil {
 		return nil, err
 	}
@@ -302,30 +295,17 @@ func (b *BlockChain) FetchTransactionStore(tx *btcwire.MsgTx) (TxStore, error) {
 		return nil, err
 	}
 
-	// Create list big
+	// Create a set of needed transactions from the transactions referenced
-	txNeededList := make([]*btcwire.ShaHash, 0, len(tx.TxIn)+1)
+	// by the inputs of the passed transaction.  Also, add the passed
-	txNeededList = append(txNeededList, &txHash)
+	// transaction itself as a way for the caller to detect duplicates.
-
+	txNeededSet := make(map[btcwire.ShaHash]bool)
-	// Loop through all of the transaction inputs collecting them into lists of what is needed and
+	txNeededSet[txHash] = true
-	// what is already known (in-flight).
-	txStore := make(TxStore)
 	for _, txIn := range tx.TxIn {
-		// Add an entry to the transaction store for the needed
+		txNeededSet[txIn.PreviousOutpoint.Hash] = true
-		// transaction with it set to missing by default.
-		originHash := &txIn.PreviousOutpoint.Hash
-		txD := &TxData{Hash: originHash, Err: btcdb.TxShaMissing}
-		txStore[*originHash] = txD
-		txNeededList = append(txNeededList, originHash)
-	}
-
-	// Request the input transactions from the point of view of the node.
-	txNeededStore := fetchTxListMain(b.db, txNeededList)
-
-	// Merge the results of the requested transactions and the in-flight
-	// transactions.
-	for _, txD := range txNeededStore {
-		txStore[*txD.Hash] = txD
 	}
 
+	// Request the input transactions from the point of view of the end of
+	// the main chain.
+	txStore := fetchTxStoreMain(b.db, txNeededSet)
 	return txStore, nil
 }

validate.go

@@ -555,7 +555,11 @@ func (b *BlockChain) checkBIP0030(node *blockNode, block *btcutil.Block) error {
 	if err != nil {
 		return nil
 	}
-	txResults, err := b.fetchTxList(node, fetchList)
+	fetchSet := make(map[btcwire.ShaHash]bool)
+	for _, txHash := range fetchList {
+		fetchSet[*txHash] = true
+	}
+	txResults, err := b.fetchTxStore(node, fetchSet)
 	if err != nil {
 		return err
 	}