Cleanup and finish BIP0037 bloom filter API.

This commit finishes the work started by @dajohi on bloom filters.

- Rename the package from bloomfilter to bloom
- Rename New function to NewFiler
- Rename Load function to LoadFilter
- Rename BloomFilter type to Filter
- Rename Contains to Matches
- Correct tx match handling to match all inputs and outputs instead of
  only the first one
- Optimize murmur hash function by using constants
- Optimize the merkle block creation and reduce num of memory allocations
  required
- Make MsgFilterLoad concurrent safe as intended
- Update various code consistency issues
- Add a lot of comments
- Improve tests
- Make the code golint clean

bloom/filter.go

@@ -0,0 +1,348 @@
+// Copyright (c) 2014 Conformal Systems LLC.
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package bloom
+
+import (
+	"encoding/binary"
+	"math"
+	"sync"
+
+	"github.com/conformal/btcscript"
+	"github.com/conformal/btcutil"
+	"github.com/conformal/btcwire"
+)
+
+// ln2Squared is simply the square of the natural log of 2.
+const ln2Squared = math.Ln2 * math.Ln2
+
+// minUint32 is a convenience function to return the minimum value of the two
+// passed uint32 values.
+func minUint32(a, b uint32) uint32 {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// Filter defines a bitcoin bloom filter that provides easy manipulation of raw
+// filter data.
+type Filter struct {
+	sync.Mutex
+	msgFilterLoad *btcwire.MsgFilterLoad
+}
+
+// NewFilter creates a new bloom filter instance, mainly to be used by SPV
+// clients.  The tweak parameter is a random value added to the seed value.
+// The false positive rate is the probability of a false positive where 1.0 is
+// "match everything" and zero is unachievable.  Thus, providing any false
+// positive rates less than 0 or greater than 1 will be adjusted to the valid
+// range.
+//
+// For more information on what values to use for both elements and fprate,
+// see https://en.wikipedia.org/wiki/Bloom_filter.
+func NewFilter(elements, tweak uint32, fprate float64, flags btcwire.BloomUpdateType) *Filter {
+	// Massage the false positive rate to sane values.
+	if fprate > 1.0 {
+		fprate = 1.0
+	}
+	if fprate < 0 {
+		fprate = 1e-9
+	}
+
+	// Calculate the size of the filter in bytes for the given number of
+	// elements and false positive rate.
+	//
+	// Equivalent to m = -(n*ln(p) / ln(2)^2), where m is in bits.
+	// Then clamp it to the maximum filter size and convert to bytes.
+	dataLen := uint32(-1 * float64(elements) * math.Log(fprate) / ln2Squared)
+	dataLen = minUint32(dataLen, btcwire.MaxFilterLoadFilterSize*8) / 8
+
+	// Calculate the number of hash functions based on the size of the
+	// filter calculated above and the number of elements.
+	//
+	// Equivalent to k = (m/n) * ln(2)
+	// Then clamp it to the maximum allowed hash funcs.
+	hashFuncs := uint32(float64(dataLen*8) / float64(elements) * math.Ln2)
+	hashFuncs = minUint32(hashFuncs, btcwire.MaxFilterLoadHashFuncs)
+
+	data := make([]byte, dataLen)
+	msg := btcwire.NewMsgFilterLoad(data, hashFuncs, tweak, flags)
+
+	return &Filter{
+		msgFilterLoad: msg,
+	}
+}
+
+// LoadFilter creates a new Filter instance with the given underlying
+// btcwire.MsgFilterLoad.
+func LoadFilter(filter *btcwire.MsgFilterLoad) *Filter {
+	return &Filter{
+		msgFilterLoad: filter,
+	}
+}
+
+// IsLoaded returns true if a filter is loaded, otherwise false.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) IsLoaded() bool {
+	bf.Lock()
+	defer bf.Unlock()
+
+	return bf.msgFilterLoad != nil
+}
+
+// Unload clears the bloom filter.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) Unload() {
+	bf.Lock()
+	defer bf.Unlock()
+
+	bf.msgFilterLoad = nil
+}
+
+// hash returns the bit offset in the bloom filter which corresponds to the
+// passed data for the given indepedent hash function number.
+func (bf *Filter) hash(hashNum uint32, data []byte) uint32 {
+	// bitcoind: 0xfba4c795 chosen as it guarantees a reasonable bit
+	// difference between hashNum values.
+	//
+	// Note that << 3 is equivalent to multiplying by 8, but is faster.
+	// Thus the returned hash is brought into range of the number of bits
+	// the filter has and returned.
+	mm := MurmurHash3(hashNum*0xfba4c795+bf.msgFilterLoad.Tweak, data)
+	return mm % (uint32(len(bf.msgFilterLoad.Filter)) << 3)
+}
+
+// matches returns true if the bloom filter might contain the passed data and
+// false if it definitely does not.
+//
+// This function MUST be called with the filter lock held.
+func (bf *Filter) matches(data []byte) bool {
+	if bf.msgFilterLoad == nil {
+		return false
+	}
+
+	// The bloom filter does not contain the data if any of the bit offsets
+	// which result from hashing the data using each independent hash
+	// function are not set.  The shifts and masks below are a faster
+	// equivalent of:
+	//   arrayIndex := idx / 8     (idx >> 3)
+	//   bitOffset := idx % 8      (idx & 7)
+	///  if filter[arrayIndex] & 1<<bitOffset == 0 { ... }
+	for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
+		idx := bf.hash(i, data)
+		if bf.msgFilterLoad.Filter[idx>>3]&(1<<(idx&7)) == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// Matches returns true if the bloom filter might contain the passed data and
+// false if it definitely does not.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) Matches(data []byte) bool {
+	bf.Lock()
+	defer bf.Unlock()
+
+	return bf.matches(data)
+}
+
+// matchesOutPoint returns true if the bloom filter might contain the passed
+// outpoint and false if it definitely does not.
+//
+// This function MUST be called with the filter lock held.
+func (bf *Filter) matchesOutPoint(outpoint *btcwire.OutPoint) bool {
+	// Serialize
+	var buf [btcwire.HashSize + 4]byte
+	copy(buf[:], outpoint.Hash.Bytes())
+	binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
+
+	return bf.matches(buf[:])
+}
+
+// MatchesOutPoint returns true if the bloom filter might contain the passed
+// outpoint and false if it definitely does not.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) MatchesOutPoint(outpoint *btcwire.OutPoint) bool {
+	bf.Lock()
+	defer bf.Unlock()
+
+	return bf.matchesOutPoint(outpoint)
+}
+
+// add adds the passed byte slice to the bloom filter.
+//
+// This function MUST be called with the filter lock held.
+func (bf *Filter) add(data []byte) {
+	if bf.msgFilterLoad == nil {
+		return
+	}
+
+	// Adding data to a bloom filter consists of setting all of the bit
+	// offsets which result from hashing the data using each independent
+	// hash function.  The shifts and masks below are a faster equivalent
+	// of:
+	//   arrayIndex := idx / 8    (idx >> 3)
+	//   bitOffset := idx % 8     (idx & 7)
+	///  filter[arrayIndex] |= 1<<bitOffset
+	for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
+		idx := bf.hash(i, data)
+		bf.msgFilterLoad.Filter[idx>>3] |= (1 << (7 & idx))
+	}
+}
+
+// Add adds the passed byte slice to the bloom filter.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) Add(data []byte) {
+	bf.Lock()
+	defer bf.Unlock()
+
+	bf.add(data)
+}
+
+// AddShaHash adds the passed btcwire.ShaHash to the Filter.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) AddShaHash(sha *btcwire.ShaHash) {
+	bf.Lock()
+	defer bf.Unlock()
+
+	bf.add(sha.Bytes())
+}
+
+// addOutPoint adds the passed transaction outpoint to the bloom filter.
+//
+// This function MUST be called with the filter lock held.
+func (bf *Filter) addOutPoint(outpoint *btcwire.OutPoint) {
+	// Serialize
+	var buf [btcwire.HashSize + 4]byte
+	copy(buf[:], outpoint.Hash.Bytes())
+	binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
+
+	bf.add(buf[:])
+}
+
+// AddOutPoint adds the passed transaction outpoint to the bloom filter.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) AddOutPoint(outpoint *btcwire.OutPoint) {
+	bf.Lock()
+	defer bf.Unlock()
+
+	bf.addOutPoint(outpoint)
+}
+
+// maybeAddOutpoint potentially adds the passed outpoint to the bloom filter
+// depending on the bloom update flags and the type of the passed public key
+// script.
+//
+// This function MUST be called with the filter lock held.
+func (bf *Filter) maybeAddOutpoint(pkScript []byte, outHash *btcwire.ShaHash, outIdx uint32) {
+	switch bf.msgFilterLoad.Flags {
+	case btcwire.BloomUpdateAll:
+		outpoint := btcwire.NewOutPoint(outHash, outIdx)
+		bf.addOutPoint(outpoint)
+	case btcwire.BloomUpdateP2PubkeyOnly:
+		class := btcscript.GetScriptClass(pkScript)
+		if class == btcscript.PubKeyTy || class == btcscript.MultiSigTy {
+			outpoint := btcwire.NewOutPoint(outHash, outIdx)
+			bf.addOutPoint(outpoint)
+		}
+	}
+}
+
+// matchTxAndUpdate returns true if the bloom filter matches data within the
+// passed transaction, otherwise false is returned.  If the filter does match
+// the passed transaction, it will also update the filter depending on the bloom
+// update flags set via the loaded filter if needed.
+//
+// This function MUST be called with the filter lock held.
+func (bf *Filter) matchTxAndUpdate(tx *btcutil.Tx) bool {
+	// Check if the filter matches the hash of the transaction.
+	// This is useful for finding transactions when they appear in a block.
+	matched := bf.matches(tx.Sha().Bytes())
+
+	// Check if the filter matches any data elements in the public key
+	// scripts of any of the outputs.  When it does, add the outpoint that
+	// matched so transactions which spend from the matched transaction are
+	// also included in the filter.  This removes the burden of updating the
+	// filter for this scenario from the client.  It is also more efficient
+	// on the network since it avoids the need for another filteradd message
+	// from the client and avoids some potential races that could otherwise
+	// occur.
+	for i, txOut := range tx.MsgTx().TxOut {
+		pushedData, err := btcscript.PushedData(txOut.PkScript)
+		if err != nil {
+			continue
+		}
+
+		for _, data := range pushedData {
+			if !bf.matches(data) {
+				continue
+			}
+
+			matched = true
+			bf.maybeAddOutpoint(txOut.PkScript, tx.Sha(), uint32(i))
+			break
+		}
+	}
+
+	// Nothing more to do if a match has already been made.
+	if matched {
+		return true
+	}
+
+	// At this point, the transaction and none of the data elements in the
+	// public key scripts of its outputs matched.
+
+	// Check if the filter matches any outpoints this transaction spends or
+	// any any data elements in the signature scripts of any of the inputs.
+	for _, txin := range tx.MsgTx().TxIn {
+		if bf.matchesOutPoint(&txin.PreviousOutpoint) {
+			return true
+		}
+
+		pushedData, err := btcscript.PushedData(txin.SignatureScript)
+		if err != nil {
+			continue
+		}
+		for _, data := range pushedData {
+			if bf.matches(data) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// MatchTxAndUpdate returns true if the bloom filter matches data within the
+// passed transaction, otherwise false is returned.  If the filter does match
+// the passed transaction, it will also update the filter depending on the bloom
+// update flags set via the loaded filter if needed.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) MatchTxAndUpdate(tx *btcutil.Tx) bool {
+	bf.Lock()
+	defer bf.Unlock()
+
+	return bf.matchTxAndUpdate(tx)
+}
+
+// MsgFilterLoad returns the underlying btcwire.MsgFilterLoad for the bloom
+// filter.
+//
+// This function is safe for concurrent access.
+func (bf *Filter) MsgFilterLoad() *btcwire.MsgFilterLoad {
+	bf.Lock()
+	defer bf.Unlock()
+
+	return bf.msgFilterLoad
+}

bloom/filter_test.go

@@ -1,24 +1,46 @@
-package bloomfilter_test
+package bloom_test
 
 import (
 	"bytes"
 	"encoding/hex"
-	"github.com/conformal/btcutil"
-	"github.com/conformal/btcutil/bloomfilter"
-	"github.com/conformal/btcwire"
 	"testing"
+
+	"github.com/conformal/btcutil"
+	"github.com/conformal/btcutil/bloom"
+	"github.com/conformal/btcwire"
 )
 
+// TestFilterLarge ensures a maximum sized filter can be created.
 func TestFilterLarge(t *testing.T) {
-	f := bloomfilter.New(100000000, 0, 0.01, btcwire.BloomUpdateNone)
+	f := bloom.NewFilter(100000000, 0, 0.01, btcwire.BloomUpdateNone)
-
 	if len(f.MsgFilterLoad().Filter) > btcwire.MaxFilterLoadFilterSize {
 		t.Errorf("TestFilterLarge test failed: %d > %d",
 			len(f.MsgFilterLoad().Filter), btcwire.MaxFilterLoadFilterSize)
 	}
 }
 
-func TestFilterInsert1(t *testing.T) {
+// TestFilterLoad ensures loading and unloading of a filter pass.
+func TestFilterLoad(t *testing.T) {
+	merkle := btcwire.MsgFilterLoad{}
+
+	f := bloom.LoadFilter(&merkle)
+	if !f.IsLoaded() {
+		t.Errorf("TestFilterLoad IsLoaded test failed: want %d got %d",
+			true, !f.IsLoaded())
+		return
+	}
+	f.Unload()
+	if f.IsLoaded() {
+		t.Errorf("TestFilterLoad IsLoaded test failed: want %d got %d",
+			f.IsLoaded(), false)
+		return
+	}
+}
+
+// TestFilterInsert ensures inserting data into the filter causes that data
+// to be matched and the resulting serialized MsgFilterLoad is the expected
+// value.
+func TestFilterInsert(t *testing.T) {
 	var tests = []struct {
 		hex    string
 		insert bool
@@ -29,46 +51,49 @@ func TestFilterInsert1(t *testing.T) {
 		{"b9300670b4c5366e95b2699e8b18bc75e5f729c5", true},
 	}
 
-	f := bloomfilter.New(3, 0, 0.01, btcwire.BloomUpdateAll)
+	f := bloom.NewFilter(3, 0, 0.01, btcwire.BloomUpdateAll)
 
-	for x, test := range tests {
+	for i, test := range tests {
 		data, err := hex.DecodeString(test.hex)
 		if err != nil {
-			t.Errorf("TestFilterInsert1 DecodeString failed: %v\n", err)
+			t.Errorf("TestFilterInsert DecodeString failed: %v\n", err)
 			return
 		}
 		if test.insert {
 			f.Add(data)
 		}
 
-		result := f.Contains(data)
+		result := f.Matches(data)
 		if test.insert != result {
-			t.Errorf("TestFilterInsert1 Contains test #%d failure: got %v want %v\n",
+			t.Errorf("TestFilterInsert Matches test #%d failure: got %v want %v\n",
-				x, result, test.insert)
+				i, result, test.insert)
 			return
 		}
 	}
 
 	want, err := hex.DecodeString("03614e9b050000000000000001")
 	if err != nil {
-		t.Errorf("TestFilterInsert1 DecodeString failed: %v\n", err)
+		t.Errorf("TestFilterInsert DecodeString failed: %v\n", err)
 		return
 	}
 
 	got := bytes.NewBuffer(nil)
 	err = f.MsgFilterLoad().BtcEncode(got, btcwire.ProtocolVersion)
 	if err != nil {
-		t.Errorf("TestFilterInsert1 BtcDecode failed: %v\n", err)
+		t.Errorf("TestFilterInsert BtcDecode failed: %v\n", err)
 		return
 	}
 
 	if !bytes.Equal(got.Bytes(), want) {
-		t.Errorf("TestFilterInsert1 failure: got %v want %v\n",
+		t.Errorf("TestFilterInsert failure: got %v want %v\n",
 			got.Bytes(), want)
 		return
 	}
 }
 
+// TestFilterInsert ensures inserting data into the filter with a tweak causes
+// that data to be matched and the resulting serialized MsgFilterLoad is the
+// expected value.
 func TestFilterInsertWithTweak(t *testing.T) {
 	var tests = []struct {
 		hex    string
@@ -80,9 +105,9 @@ func TestFilterInsertWithTweak(t *testing.T) {
 		{"b9300670b4c5366e95b2699e8b18bc75e5f729c5", true},
 	}
 
-	f := bloomfilter.New(3, 2147483649, 0.01, btcwire.BloomUpdateAll)
+	f := bloom.NewFilter(3, 2147483649, 0.01, btcwire.BloomUpdateAll)
 
-	for x, test := range tests {
+	for i, test := range tests {
 		data, err := hex.DecodeString(test.hex)
 		if err != nil {
 			t.Errorf("TestFilterInsertWithTweak DecodeString failed: %v\n", err)
@@ -92,10 +117,10 @@ func TestFilterInsertWithTweak(t *testing.T) {
 			f.Add(data)
 		}
 
-		result := f.Contains(data)
+		result := f.Matches(data)
 		if test.insert != result {
-			t.Errorf("TestFilterInsertWithTweak Contains test #%d failure: got %v want %v\n",
+			t.Errorf("TestFilterInsertWithTweak Matches test #%d failure: got %v want %v\n",
-				x, result, test.insert)
+				i, result, test.insert)
 			return
 		}
 	}
@@ -119,6 +144,8 @@ func TestFilterInsertWithTweak(t *testing.T) {
 	}
 }
 
+// TestFilterInsertKey ensures inserting public keys and addresses works as
+// expected.
 func TestFilterInsertKey(t *testing.T) {
 	secret := "5Kg1gnAjaLfKiwhhPpGS3QfRg2m6awQvaj98JCZBZQ5SuS2F15C"
 
@@ -128,8 +155,7 @@ func TestFilterInsertKey(t *testing.T) {
 		return
 	}
 
-
+	f := bloom.NewFilter(2, 0, 0.001, btcwire.BloomUpdateAll)
-	f := bloomfilter.New(2, 0, 0.001, btcwire.BloomUpdateAll)
 	f.Add(wif.SerializePubKey())
 	f.Add(btcutil.Hash160(wif.SerializePubKey()))
 
@@ -208,7 +234,7 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 
-	f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr := "b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b"
 	sha, err := btcwire.NewShaHashFromStr(inputStr)
 	if err != nil {
@@ -216,11 +242,11 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.AddShaHash(sha)
-	if !f.MatchesTx(tx) {
+	if !f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch didn't match sha %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "6bff7fcd4f8565ef406dd5d63d4ff94f318fe82027fd4dc451b04474019f74b4"
 	shaBytes, err := hex.DecodeString(inputStr)
 	if err != nil {
@@ -228,11 +254,11 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.Add(shaBytes)
-	if !f.MatchesTx(tx) {
+	if !f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch didn't match sha %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "30450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065" +
 		"f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643" +
 		"ac4cb7cb3c462aced7f14711a01"
@@ -242,11 +268,11 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.Add(shaBytes)
-	if !f.MatchesTx(tx) {
+	if !f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch didn't match input signature %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95" +
 		"c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe" +
 		"76036c339"
@@ -256,11 +282,11 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.Add(shaBytes)
-	if !f.MatchesTx(tx) {
+	if !f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch didn't match input pubkey %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "04943fdd508053c75000106d3bc6e2754dbcff19"
 	shaBytes, err = hex.DecodeString(inputStr)
 	if err != nil {
@@ -268,14 +294,14 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.Add(shaBytes)
-	if !f.MatchesTx(tx) {
+	if !f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch didn't match output address %s", inputStr)
 	}
-	if !f.MatchesTx(spendingTx) {
+	if !f.MatchTxAndUpdate(spendingTx) {
 		t.Errorf("TestFilterBloomMatch spendingTx didn't match output address %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "a266436d2965547608b9e15d9032a7b9d64fa431"
 	shaBytes, err = hex.DecodeString(inputStr)
 	if err != nil {
@@ -283,11 +309,11 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.Add(shaBytes)
-	if !f.MatchesTx(tx) {
+	if !f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch didn't match output address %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"
 	sha, err = btcwire.NewShaHashFromStr(inputStr)
 	if err != nil {
@@ -296,11 +322,11 @@ func TestFilterBloomMatch(t *testing.T) {
 	}
 	outpoint := btcwire.NewOutPoint(sha, 0)
 	f.AddOutPoint(outpoint)
-	if !f.MatchesTx(tx) {
+	if !f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch didn't match outpoint %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "00000009e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436"
 	sha, err = btcwire.NewShaHashFromStr(inputStr)
 	if err != nil {
@@ -308,11 +334,11 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.AddShaHash(sha)
-	if f.MatchesTx(tx) {
+	if f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch matched sha %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "0000006d2965547608b9e15d9032a7b9d64fa431"
 	shaBytes, err = hex.DecodeString(inputStr)
 	if err != nil {
@@ -320,11 +346,11 @@ func TestFilterBloomMatch(t *testing.T) {
 		return
 	}
 	f.Add(shaBytes)
-	if f.MatchesTx(tx) {
+	if f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch matched address %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"
 	sha, err = btcwire.NewShaHashFromStr(inputStr)
 	if err != nil {
@@ -333,11 +359,11 @@ func TestFilterBloomMatch(t *testing.T) {
 	}
 	outpoint = btcwire.NewOutPoint(sha, 1)
 	f.AddOutPoint(outpoint)
-	if f.MatchesTx(tx) {
+	if f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch matched outpoint %s", inputStr)
 	}
 
-	f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 	inputStr = "000000d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"
 	sha, err = btcwire.NewShaHashFromStr(inputStr)
 	if err != nil {
@@ -346,13 +372,13 @@ func TestFilterBloomMatch(t *testing.T) {
 	}
 	outpoint = btcwire.NewOutPoint(sha, 0)
 	f.AddOutPoint(outpoint)
-	if f.MatchesTx(tx) {
+	if f.MatchTxAndUpdate(tx) {
 		t.Errorf("TestFilterBloomMatch matched outpoint %s", inputStr)
 	}
 }
 
 func TestFilterInsertUpdateNone(t *testing.T) {
-	f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateNone)
+	f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateNone)
 
 	// Add the generation pubkey
 	inputStr := "04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c" +
@@ -382,7 +408,7 @@ func TestFilterInsertUpdateNone(t *testing.T) {
 	}
 	outpoint := btcwire.NewOutPoint(sha, 0)
 
-	if f.ContainsOutPoint(outpoint) {
+	if f.MatchesOutPoint(outpoint) {
 		t.Errorf("TestFilterInsertUpdateNone matched outpoint %s", inputStr)
 		return
 	}
@@ -395,7 +421,7 @@ func TestFilterInsertUpdateNone(t *testing.T) {
 	}
 	outpoint = btcwire.NewOutPoint(sha, 0)
 
-	if f.ContainsOutPoint(outpoint) {
+	if f.MatchesOutPoint(outpoint) {
 		t.Errorf("TestFilterInsertUpdateNone matched outpoint %s", inputStr)
 		return
 	}
@@ -500,7 +526,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
 		return
 	}
 
-	f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateP2PubkeyOnly)
+	f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateP2PubkeyOnly)
 
 	// Generation pubkey
 	inputStr := "04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c" +
@@ -523,7 +549,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
 	f.Add(inputBytes)
 
 	// Ignore return value -- this is just used to update the filter.
-	_, _ = bloomfilter.NewMerkleBlock(block, f)
+	_, _ = bloom.NewMerkleBlock(block, f)
 
 	// We should match the generation pubkey
 	inputStr = "147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"
@@ -533,7 +559,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
 		return
 	}
 	outpoint := btcwire.NewOutPoint(sha, 0)
-	if !f.ContainsOutPoint(outpoint) {
+	if !f.MatchesOutPoint(outpoint) {
 		t.Errorf("TestMerkleBlockP2PubKeyOnly didn't match the generation "+
 			"outpoint %s", inputStr)
 		return
@@ -547,7 +573,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
 		return
 	}
 	outpoint = btcwire.NewOutPoint(sha, 0)
-	if f.ContainsOutPoint(outpoint) {
+	if f.MatchesOutPoint(outpoint) {
 		t.Errorf("TestMerkleBlockP2PubKeyOnly matched outpoint %s", inputStr)
 		return
 	}

bloom/merkleblock.go

@@ -0,0 +1,120 @@
+package bloom
+
+import (
+	"github.com/conformal/btcchain"
+	"github.com/conformal/btcutil"
+	"github.com/conformal/btcwire"
+)
+
+// merkleBlock is used to house intermediate information needed to generate a
+// btcwire.MsgMerkleBlock according to a filter.
+type merkleBlock struct {
+	numTx       uint32
+	allHashes   []*btcwire.ShaHash
+	finalHashes []*btcwire.ShaHash
+	matchedBits []byte
+	bits        []byte
+}
+
+// calcTreeWidth calculates and returns the the number of nodes (width) or a
+// merkle tree at the given depth-first height.
+func (m *merkleBlock) calcTreeWidth(height uint32) uint32 {
+	return (m.numTx + (1 << height) - 1) >> height
+}
+
+// calcHash returns the hash for a sub-tree given a depth-first height and
+// node position.
+func (m *merkleBlock) calcHash(height, pos uint32) *btcwire.ShaHash {
+	if height == 0 {
+		return m.allHashes[pos]
+	}
+
+	var right *btcwire.ShaHash
+	left := m.calcHash(height-1, pos*2)
+	if pos*2+1 < m.calcTreeWidth(height-1) {
+		right = m.calcHash(height-1, pos*2+1)
+	} else {
+		right = left
+	}
+	return btcchain.HashMerkleBranches(left, right)
+}
+
+// traverseAndBuild builds a partial merkle tree using a recursive depth-first
+// approach.  As it calculates the hashes, it also saves whether or not each
+// node is a parent node and a list of final hashes to be included in the
+// merkle block.
+func (m *merkleBlock) traverseAndBuild(height, pos uint32) {
+	// Determine whether this node is a parent of a matched node.
+	var isParent byte
+	for i := pos << height; i < (pos+1)<<height && i < m.numTx; i++ {
+		isParent |= m.matchedBits[i]
+	}
+	m.bits = append(m.bits, isParent)
+
+	// When the node is a leaf node or not a parent of a matched node,
+	// append the hash to the list that will be part of the final merkle
+	// block.
+	if height == 0 || isParent == 0x00 {
+		m.finalHashes = append(m.finalHashes, m.calcHash(height, pos))
+		return
+	}
+
+	// At this point, the node is an internal node and it is the parent of
+	// of an included leaf node.
+
+	// Descend into the left child and process its sub-tree.
+	m.traverseAndBuild(height-1, pos*2)
+
+	// Descend into the right child and process its sub-tree if
+	// there is one.
+	if pos*2+1 < m.calcTreeWidth(height-1) {
+		m.traverseAndBuild(height-1, pos*2+1)
+	}
+}
+
+// NewMerkleBlock returns a new *btcwire.MsgMerkleBlock based on the passed
+// block and filter.
+func NewMerkleBlock(block *btcutil.Block, filter *Filter) (*btcwire.MsgMerkleBlock, []*btcwire.ShaHash) {
+	numTx := uint32(len(block.Transactions()))
+	mBlock := merkleBlock{
+		numTx:       numTx,
+		allHashes:   make([]*btcwire.ShaHash, 0, numTx),
+		matchedBits: make([]byte, 0, numTx),
+	}
+
+	// Find and keep track of any transactions that match the filter.
+	var matchedHashes []*btcwire.ShaHash
+	for _, tx := range block.Transactions() {
+		if filter.MatchTxAndUpdate(tx) {
+			mBlock.matchedBits = append(mBlock.matchedBits, 0x01)
+			matchedHashes = append(matchedHashes, tx.Sha())
+		} else {
+			mBlock.matchedBits = append(mBlock.matchedBits, 0x00)
+		}
+		mBlock.allHashes = append(mBlock.allHashes, tx.Sha())
+	}
+
+	// Calculate the number of merkle branches (height) in the tree.
+	height := uint32(0)
+	for mBlock.calcTreeWidth(height) > 1 {
+		height++
+	}
+
+	// Build the depth-first partial merkle tree.
+	mBlock.traverseAndBuild(height, 0)
+
+	// Create and return the merkle block.
+	msgMerkleBlock := btcwire.MsgMerkleBlock{
+		Header:       block.MsgBlock().Header,
+		Transactions: uint32(mBlock.numTx),
+		Hashes:       make([]*btcwire.ShaHash, 0, len(mBlock.finalHashes)),
+		Flags:        make([]byte, (len(mBlock.bits)+7)/8),
+	}
+	for _, sha := range mBlock.finalHashes {
+		msgMerkleBlock.AddTxHash(sha)
+	}
+	for i := uint32(0); i < uint32(len(mBlock.bits)); i++ {
+		msgMerkleBlock.Flags[i/8] |= mBlock.bits[i] << (i % 8)
+	}
+	return &msgMerkleBlock, matchedHashes
+}

bloom/merkleblock_test.go

@@ -1,12 +1,13 @@
-package bloomfilter_test
+package bloom_test
 
 import (
 	"bytes"
 	"encoding/hex"
-	"github.com/conformal/btcutil"
-	"github.com/conformal/btcutil/bloomfilter"
-	"github.com/conformal/btcwire"
 	"testing"
+
+	"github.com/conformal/btcutil"
+	"github.com/conformal/btcutil/bloom"
+	"github.com/conformal/btcwire"
 )
 
 func TestMerkleBlock3(t *testing.T) {
@@ -29,7 +30,7 @@ func TestMerkleBlock3(t *testing.T) {
 		return
 	}
 
-	f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
+	f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
 
 	inputStr := "63194f18be0af63f2c6bc9dc0f777cbefed3d9415c4af83f3ee3a3d669c00cb5"
 	sha, err := btcwire.NewShaHashFromStr(inputStr)
@@ -40,7 +41,7 @@ func TestMerkleBlock3(t *testing.T) {
 
 	f.AddShaHash(sha)
 
-	mBlock, _ := bloomfilter.NewMerkleBlock(blk, f)
+	mBlock, _ := bloom.NewMerkleBlock(blk, f)
 
 	wantStr := "0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4" +
 		"b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc" +

bloom/murmurhash3.go

@@ -0,0 +1,68 @@
+package bloom
+
+import (
+	"encoding/binary"
+)
+
+// The following constants are used by the MurmurHash3 algorithm.
+const (
+	murmurC1 = 0xcc9e2d51
+	murmurC2 = 0x1b873593
+	murmurR1 = 15
+	murmurR2 = 13
+	murmurM  = 5
+	murmurN  = 0xe6546b64
+)
+
+// MurmurHash3 implements a non-cryptographic hash function using the
+// MurmurHash3 algorithm.  This implementation yields a 32-bit hash value which
+// is suitable for general hash-based lookups.  The seed can be used to
+// effectively randomize the hash function.  This makes it ideal for use in
+// bloom filters which need multiple independent hash functions.
+func MurmurHash3(seed uint32, data []byte) uint32 {
+	dataLen := uint32(len(data))
+	hash := seed
+	k := uint32(0)
+	numBlocks := dataLen / 4
+
+	// Calculate the hash in 4-byte chunks.
+	for i := uint32(0); i < numBlocks; i++ {
+		k = binary.LittleEndian.Uint32(data[i*4:])
+		k *= murmurC1
+		k = (k << murmurR1) | (k >> (32 - murmurR1))
+		k *= murmurC2
+
+		hash ^= k
+		hash = (hash << murmurR2) | (hash >> (32 - murmurR2))
+		hash = hash*murmurM + murmurN
+	}
+
+	// Handle remaining bytes.
+	tailIdx := numBlocks * 4
+	k = 0
+
+	switch dataLen & 3 {
+	case 3:
+		k ^= uint32(data[tailIdx+2]) << 16
+		fallthrough
+	case 2:
+		k ^= uint32(data[tailIdx+1]) << 8
+		fallthrough
+	case 1:
+		k ^= uint32(data[tailIdx])
+		k *= murmurC1
+		k = (k << murmurR1) | (k >> (32 - murmurR1))
+		k *= murmurC2
+		hash ^= k
+	}
+
+	// Finalization.
+	hash ^= uint32(dataLen)
+	hash ^= hash >> 16
+	hash *= 0x85ebca6b
+	hash ^= hash >> 13
+	hash *= 0xc2b2ae35
+	hash ^= hash >> 16
+
+	return hash
+}

bloom/murmurhash3_test.go

@@ -1,10 +1,13 @@
-package bloomfilter_test
+package bloom_test
 
 import (
-	"github.com/conformal/btcutil/bloomfilter"
 	"testing"
+
+	"github.com/conformal/btcutil/bloom"
 )
 
+// TestMurmurHash3 ensure the MurmurHash3 function produces the correct hash
+// when given various seeds and data.
 func TestMurmurHash3(t *testing.T) {
 	var tests = []struct {
 		seed uint32
@@ -27,11 +30,11 @@ func TestMurmurHash3(t *testing.T) {
 		{0x00000000, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88}, 0xb4698def},
 	}
 
-	for x, test := range tests {
+	for i, test := range tests {
-		result := bloomfilter.MurmurHash3(test.seed, test.data)
+		result := bloom.MurmurHash3(test.seed, test.data)
 		if result != test.out {
 			t.Errorf("MurmurHash3 test #%d failed: got %v want %v\n",
-				x, result, test.out)
+				i, result, test.out)
 			continue
 		}
 	}

bloom/test_coverage.txt

@@ -0,0 +1,27 @@
+
+github.com/conformal/btcutil/bloom/murmurhash3.go	 MurmurHash3			 100.00% (31/31)
+github.com/conformal/btcutil/bloom/merkleblock.go	 NewMerkleBlock			 100.00% (22/22)
+github.com/conformal/btcutil/bloom/merkleblock.go	 merkleBlock.traverseAndBuild	 100.00% (9/9)
+github.com/conformal/btcutil/bloom/merkleblock.go	 merkleBlock.calcHash		 100.00% (8/8)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.maybeAddOutpoint	 100.00% (7/7)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.matchesOutPoint		 100.00% (4/4)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.addOutPoint		 100.00% (4/4)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.AddShaHash		 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.IsLoaded		 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.Unload			 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.Matches			 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.MatchesOutPoint		 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.Add			 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.AddOutPoint		 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.MatchTxAndUpdate	 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.MsgFilterLoad		 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 minUint32			 100.00% (3/3)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.hash			 100.00% (2/2)
+github.com/conformal/btcutil/bloom/filter.go		 LoadFilter			 100.00% (1/1)
+github.com/conformal/btcutil/bloom/merkleblock.go	 merkleBlock.calcTreeWidth	 100.00% (1/1)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.matchTxAndUpdate	 91.30% (21/23)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.matches			 85.71% (6/7)
+github.com/conformal/btcutil/bloom/filter.go		 NewFilter			 81.82% (9/11)
+github.com/conformal/btcutil/bloom/filter.go		 Filter.add			 80.00% (4/5)
+github.com/conformal/btcutil/bloom			 ----------------------------	 96.36% (159/165)
+

bloomfilter/filter.go

@@ -1,225 +0,0 @@
-// Copyright (c) 2014 Conformal Systems LLC.
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package bloomfilter
-
-import (
-	"encoding/binary"
-	"github.com/conformal/btcscript"
-	"github.com/conformal/btcutil"
-	"github.com/conformal/btcwire"
-	"math"
-	"sync"
-)
-
-// BloomFilter defines a bitcoin bloomfilter the provides easy manipulation of raw
-// filter data.
-type BloomFilter struct {
-	sync.Mutex
-	msgFilterLoad *btcwire.MsgFilterLoad
-}
-
-// New creates a new Filter instance, mainly to be used by SPV clients.  The tweak parameter is
-// a random value added to the seed value.  For more information on what values to use for both
-// elements and fprate, please see https://en.wikipedia.org/wiki/Bloom_filter.
-func New(elements, tweak uint32, fprate float64, flags btcwire.BloomUpdateType) *BloomFilter {
-	dataLen := uint32(math.Abs(math.Log(fprate)) * float64(elements) / (math.Ln2 * math.Ln2))
-	dataLen = min(dataLen, btcwire.MaxFilterLoadFilterSize*8) / 8
-
-	hashFuncs := min(uint32(float64(dataLen)*8.0/float64(elements)*math.Ln2), btcwire.MaxFilterLoadHashFuncs)
-
-	data := make([]byte, dataLen)
-	msg := btcwire.NewMsgFilterLoad(data, hashFuncs, tweak, flags)
-
-	return &BloomFilter{
-		msgFilterLoad: msg,
-	}
-}
-
-// Load creates a new BloomFilter instance with the given btcwire.MsgFilterLoad.
-func Load(filter *btcwire.MsgFilterLoad) *BloomFilter {
-	return &BloomFilter{
-		msgFilterLoad: filter,
-	}
-}
-
-// Loaded returns true if a filter is loaded, otherwise false.
-func (bf *BloomFilter) IsLoaded() bool {
-	bf.Lock()
-	defer bf.Unlock()
-
-	return bf.msgFilterLoad != nil
-}
-
-// Unload clears the Filter.
-func (bf *BloomFilter) Unload() {
-	bf.Lock()
-	defer bf.Unlock()
-
-	bf.msgFilterLoad = nil
-}
-
-func (bf *BloomFilter) contains(data []byte) bool {
-	if bf.msgFilterLoad == nil {
-		return false
-	}
-
-	for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
-		idx := bf.hash(i, data)
-		if bf.msgFilterLoad.Filter[idx>>3]&(1<<(7&idx)) != 1<<(7&idx) {
-			return false
-		}
-	}
-	return true
-}
-
-// Contains returns true if the BloomFilter contains the passed byte slice. Otherwise,
-// it returns false.
-func (bf *BloomFilter) Contains(data []byte) bool {
-	bf.Lock()
-	defer bf.Unlock()
-
-	return bf.contains(data)
-}
-
-func (bf *BloomFilter) containsOutPoint(outpoint *btcwire.OutPoint) bool {
-	// Serialize
-	var buf [btcwire.HashSize + 4]byte
-	copy(buf[:], outpoint.Hash.Bytes())
-	binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
-
-	return bf.contains(buf[:])
-}
-
-// ContainsOutPoint returns true if the BloomFilter contains the given
-// btcwire.OutPoint.  Otherwise, it returns false.
-func (bf *BloomFilter) ContainsOutPoint(outpoint *btcwire.OutPoint) bool {
-	bf.Lock()
-	defer bf.Unlock()
-
-	return bf.containsOutPoint(outpoint)
-}
-
-func (bf *BloomFilter) add(data []byte) {
-	if bf.msgFilterLoad == nil {
-		return
-	}
-
-	for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
-		idx := bf.hash(i, data)
-		bf.msgFilterLoad.Filter[idx>>3] |= (1 << (7 & idx))
-	}
-}
-
-// Add adds the passed byte slice to the BloomFilter.
-func (bf *BloomFilter) Add(data []byte) {
-	bf.Lock()
-	defer bf.Unlock()
-
-	bf.add(data)
-}
-
-// AddShaHash adds the passed btcwire.ShaHash to the BloomFilter.
-func (bf *BloomFilter) AddShaHash(sha *btcwire.ShaHash) {
-	bf.Lock()
-	defer bf.Unlock()
-
-	bf.add(sha.Bytes())
-}
-
-// AddOutPoint adds the passed btcwire.OutPoint to the BloomFilter.
-func (bf *BloomFilter) AddOutPoint(outpoint *btcwire.OutPoint) {
-	bf.Lock()
-	defer bf.Unlock()
-
-	bf.addOutPoint(outpoint)
-}
-
-func (bf *BloomFilter) addOutPoint(outpoint *btcwire.OutPoint) {
-	// Serialize
-	var buf [btcwire.HashSize + 4]byte
-	copy(buf[:], outpoint.Hash.Bytes())
-	binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
-
-	bf.add(buf[:])
-}
-
-func (bf *BloomFilter) matches(tx *btcutil.Tx) bool {
-	hash := tx.Sha().Bytes()
-	matched := bf.contains(hash)
-
-	for i, txout := range tx.MsgTx().TxOut {
-		pushedData, err := btcscript.PushedData(txout.PkScript)
-		if err != nil {
-			break
-		}
-		for _, p := range pushedData {
-			if bf.contains(p) {
-				switch bf.msgFilterLoad.Flags {
-				case btcwire.BloomUpdateAll:
-					outpoint := btcwire.NewOutPoint(tx.Sha(), uint32(i))
-					bf.addOutPoint(outpoint)
-				case btcwire.BloomUpdateP2PubkeyOnly:
-					class := btcscript.GetScriptClass(txout.PkScript)
-					if class == btcscript.PubKeyTy || class == btcscript.MultiSigTy {
-						outpoint := btcwire.NewOutPoint(tx.Sha(), uint32(i))
-						bf.addOutPoint(outpoint)
-					}
-				}
-				return true
-			}
-		}
-	}
-
-	if matched {
-		return true
-	}
-
-	for _, txin := range tx.MsgTx().TxIn {
-		if bf.containsOutPoint(&txin.PreviousOutpoint) {
-			return true
-		}
-		pushedData, err := btcscript.PushedData(txin.SignatureScript)
-		if err != nil {
-			break
-		}
-		for _, p := range pushedData {
-			if bf.contains(p) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-// MatchesTx returns true if the BloomFilter matches data within the passed transaction,
-// otherwise false is returned.  If the BloomFilter does match the passed transaction,
-// it will also update the BloomFilter if required.
-func (bf *BloomFilter) MatchesTx(tx *btcutil.Tx) bool {
-	bf.Lock()
-	defer bf.Unlock()
-
-	return bf.matches(tx)
-}
-
-// MsgFilterLoad returns the underlying btcwire.MsgFilterLoad for the BloomFilter.
-func (bf *BloomFilter) MsgFilterLoad() *btcwire.MsgFilterLoad {
-	return bf.msgFilterLoad
-}
-
-func (bf *BloomFilter) hash(hashNum uint32, data []byte) uint32 {
-	// bitcoind: 0xFBA4C795 chosen as it guarantees a reasonable bit
-	// difference between nHashNum values.
-	mm := MurmurHash3(hashNum*0xFBA4C795+bf.msgFilterLoad.Tweak, data)
-	return mm % (uint32(len(bf.msgFilterLoad.Filter)) * 8)
-}
-
-// min is a convenience function to return the minimum value of the two
-// passed uint32 values.
-func min(a, b uint32) uint32 {
-	if a < b {
-		return a
-	}
-	return b
-}

bloomfilter/merkleblock.go

@@ -1,89 +0,0 @@
-package bloomfilter
-
-import (
-	"github.com/conformal/btcchain"
-	"github.com/conformal/btcutil"
-	"github.com/conformal/btcwire"
-)
-
-type merkleBlock struct {
-	msgMerkleBlock *btcwire.MsgMerkleBlock
-	allHashes      []*btcwire.ShaHash
-	finalHashes    []*btcwire.ShaHash
-	matchedBits    []byte
-	bits           []byte
-}
-
-// NewMerkleBlock returns a new *btcwire.MsgMerkleBlock based on the passed
-// block and filter.
-func NewMerkleBlock(block *btcutil.Block, filter *BloomFilter) (*btcwire.MsgMerkleBlock, []*btcwire.ShaHash) {
-	blockHeader := block.MsgBlock().Header
-	mBlock := merkleBlock{
-		msgMerkleBlock: btcwire.NewMsgMerkleBlock(&blockHeader),
-	}
-
-	var matchedHashes []*btcwire.ShaHash
-	for _, tx := range block.Transactions() {
-		if filter.MatchesTx(tx) {
-			mBlock.matchedBits = append(mBlock.matchedBits, 0x01)
-			matchedHashes = append(matchedHashes, tx.Sha())
-		} else {
-			mBlock.matchedBits = append(mBlock.matchedBits, 0x00)
-		}
-		mBlock.allHashes = append(mBlock.allHashes, tx.Sha())
-	}
-	mBlock.msgMerkleBlock.Transactions = uint32(len(block.Transactions()))
-
-	height := uint32(0)
-	for mBlock.calcTreeWidth(height) > 1 {
-		height++
-	}
-
-	mBlock.traverseAndBuild(height, 0)
-
-	for _, sha := range mBlock.finalHashes {
-		mBlock.msgMerkleBlock.AddTxHash(sha)
-	}
-	mBlock.msgMerkleBlock.Flags = make([]byte, (len(mBlock.bits)+7)/8)
-	for i := uint32(0); i < uint32(len(mBlock.bits)); i++ {
-		mBlock.msgMerkleBlock.Flags[i/8] |= mBlock.bits[i] << (i % 8)
-	}
-	return mBlock.msgMerkleBlock, matchedHashes
-}
-
-func (m *merkleBlock) calcTreeWidth(height uint32) uint32 {
-	return (m.msgMerkleBlock.Transactions + (1 << height) - 1) >> height
-}
-
-func (m *merkleBlock) calcHash(height, pos uint32) *btcwire.ShaHash {
-	if height == 0 {
-		return m.allHashes[pos]
-	} else {
-		var right *btcwire.ShaHash
-		left := m.calcHash(height-1, pos*2)
-		if pos*2+1 < m.calcTreeWidth(height-1) {
-			right = m.calcHash(height-1, pos*2+1)
-		} else {
-			right = left
-		}
-		return btcchain.HashMerkleBranches(left, right)
-	}
-}
-
-func (m *merkleBlock) traverseAndBuild(height, pos uint32) {
-	var isParent byte
-
-	for i := pos << height; i < (pos+1)<<height && i < m.msgMerkleBlock.Transactions; i++ {
-		isParent |= m.matchedBits[i]
-	}
-
-	m.bits = append(m.bits, isParent)
-	if height == 0 || isParent == 0x00 {
-		m.finalHashes = append(m.finalHashes, m.calcHash(height, pos))
-	} else {
-		m.traverseAndBuild(height-1, pos*2)
-		if pos*2+1 < m.calcTreeWidth(height-1) {
-			m.traverseAndBuild(height-1, pos*2+1)
-		}
-	}
-}

bloomfilter/murmurhash3.go

@@ -1,55 +0,0 @@
-package bloomfilter
-
-import (
-	"encoding/binary"
-)
-
-// MurmurHash3 implements the MurmurHash3 algorithm.
-func MurmurHash3(seed uint32, data []byte) uint32 {
-	dataLen := len(data)
-	h1 := seed
-	c1 := uint32(0xcc9e2d51)
-	c2 := uint32(0x1b873593)
-	k1 := uint32(0)
-	numBlocks := dataLen / 4
-
-	// body
-	for i := 0; i < numBlocks; i++ {
-		k1 = binary.LittleEndian.Uint32(data[i * 4:])
-		k1 *= c1
-		k1 = (k1 << 15) | (k1 >> (32 - 15))
-		k1 *= c2
-		h1 ^= k1
-		h1 = (h1 << 13) | (h1 >> (32 - 13))
-		h1 = h1*5 + 0xe6546b64
-	}
-
-	// tail
-	tailidx := numBlocks * 4
-	k1 = 0
-
-	switch dataLen & 3 {
-	case 3:
-		k1 ^= uint32(data[tailidx+2]) << 16
-		fallthrough
-	case 2:
-		k1 ^= uint32(data[tailidx+1]) << 8
-		fallthrough
-	case 1:
-		k1 ^= uint32(data[tailidx])
-		k1 *= c1
-		k1 = (k1 << 15) | (k1 >> (32 - 15))
-		k1 *= c2
-		h1 ^= k1
-	}
-
-	// Finalization
-	h1 ^= uint32(dataLen)
-	h1 ^= h1 >> 16
-	h1 *= 0x85ebca6b
-	h1 ^= h1 >> 13
-	h1 *= 0xc2b2ae35
-	h1 ^= h1 >> 16
-
-	return h1
-}

bloomfilter/test_coverage.txt

@@ -1,26 +0,0 @@
-
-github.com/conformal/btcutil/bloomfilter/murmurhash3.go	 MurmurHash3			 100.00% (34/34)
-github.com/conformal/btcutil/bloomfilter/merkleblock.go	 NewMerkleBlock			 100.00% (20/20)
-github.com/conformal/btcutil/bloomfilter/merkleblock.go	 merkleBlock.traverseAndBuild	 100.00% (9/9)
-github.com/conformal/btcutil/bloomfilter/merkleblock.go	 merkleBlock.calcHash		 100.00% (8/8)
-github.com/conformal/btcutil/bloomfilter/filter.go	 New				 100.00% (6/6)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.containsOutPoint	 100.00% (4/4)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.addOutPoint	 100.00% (4/4)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.Add		 100.00% (3/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.AddShaHash		 100.00% (3/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.Contains		 100.00% (3/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.ContainsOutPoint	 100.00% (3/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.MatchesTx		 100.00% (3/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 min				 100.00% (3/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.AddOutPoint	 100.00% (3/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.hash		 100.00% (2/2)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.MsgFilterLoad	 100.00% (1/1)
-github.com/conformal/btcutil/bloomfilter/merkleblock.go	 merkleBlock.calcTreeWidth	 100.00% (1/1)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.matches		 92.86% (26/28)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.contains		 85.71% (6/7)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.add		 80.00% (4/5)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.Unload		 0.00% (0/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 BloomFilter.IsLoaded		 0.00% (0/3)
-github.com/conformal/btcutil/bloomfilter/filter.go	 Load				 0.00% (0/1)
-github.com/conformal/btcutil/bloomfilter		 ----------------------------	 92.99% (146/157)
-