Cleanup and finish BIP0037 bloom filter API.
This commit finishes the work started by @dajohi on bloom filters. - Rename the package from bloomfilter to bloom - Rename New function to NewFiler - Rename Load function to LoadFilter - Rename BloomFilter type to Filter - Rename Contains to Matches - Correct tx match handling to match all inputs and outputs instead of only the first one - Optimize murmur hash function by using constants - Optimize the merkle block creation and reduce num of memory allocations required - Make MsgFilterLoad concurrent safe as intended - Update various code consistency issues - Add a lot of comments - Improve tests - Make the code golint clean
This commit is contained in:
Dave Collins
parent
9e3269e67c
commit
ad004c0534
12 changed files with 657 additions and 459 deletions
348
bloom/filter.go
Normal file
348
bloom/filter.go
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
// Copyright (c) 2014 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package bloom
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/conformal/btcscript"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
// ln2Squared is simply the square of the natural log of 2.
|
||||
const ln2Squared = math.Ln2 * math.Ln2
|
||||
|
||||
// minUint32 is a convenience function to return the minimum value of the two
|
||||
// passed uint32 values.
|
||||
func minUint32(a, b uint32) uint32 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Filter defines a bitcoin bloom filter that provides easy manipulation of raw
|
||||
// filter data.
|
||||
type Filter struct {
|
||||
sync.Mutex
|
||||
msgFilterLoad *btcwire.MsgFilterLoad
|
||||
}
|
||||
|
||||
// NewFilter creates a new bloom filter instance, mainly to be used by SPV
|
||||
// clients. The tweak parameter is a random value added to the seed value.
|
||||
// The false positive rate is the probability of a false positive where 1.0 is
|
||||
// "match everything" and zero is unachievable. Thus, providing any false
|
||||
// positive rates less than 0 or greater than 1 will be adjusted to the valid
|
||||
// range.
|
||||
//
|
||||
// For more information on what values to use for both elements and fprate,
|
||||
// see https://en.wikipedia.org/wiki/Bloom_filter.
|
||||
func NewFilter(elements, tweak uint32, fprate float64, flags btcwire.BloomUpdateType) *Filter {
|
||||
// Massage the false positive rate to sane values.
|
||||
if fprate > 1.0 {
|
||||
fprate = 1.0
|
||||
}
|
||||
if fprate < 0 {
|
||||
fprate = 1e-9
|
||||
}
|
||||
|
||||
// Calculate the size of the filter in bytes for the given number of
|
||||
// elements and false positive rate.
|
||||
//
|
||||
// Equivalent to m = -(n*ln(p) / ln(2)^2), where m is in bits.
|
||||
// Then clamp it to the maximum filter size and convert to bytes.
|
||||
dataLen := uint32(-1 * float64(elements) * math.Log(fprate) / ln2Squared)
|
||||
dataLen = minUint32(dataLen, btcwire.MaxFilterLoadFilterSize*8) / 8
|
||||
|
||||
// Calculate the number of hash functions based on the size of the
|
||||
// filter calculated above and the number of elements.
|
||||
//
|
||||
// Equivalent to k = (m/n) * ln(2)
|
||||
// Then clamp it to the maximum allowed hash funcs.
|
||||
hashFuncs := uint32(float64(dataLen*8) / float64(elements) * math.Ln2)
|
||||
hashFuncs = minUint32(hashFuncs, btcwire.MaxFilterLoadHashFuncs)
|
||||
|
||||
data := make([]byte, dataLen)
|
||||
msg := btcwire.NewMsgFilterLoad(data, hashFuncs, tweak, flags)
|
||||
|
||||
return &Filter{
|
||||
msgFilterLoad: msg,
|
||||
}
|
||||
}
|
||||
|
||||
// LoadFilter creates a new Filter instance with the given underlying
|
||||
// btcwire.MsgFilterLoad.
|
||||
func LoadFilter(filter *btcwire.MsgFilterLoad) *Filter {
|
||||
return &Filter{
|
||||
msgFilterLoad: filter,
|
||||
}
|
||||
}
|
||||
|
||||
// IsLoaded returns true if a filter is loaded, otherwise false.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) IsLoaded() bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.msgFilterLoad != nil
|
||||
}
|
||||
|
||||
// Unload clears the bloom filter.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) Unload() {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.msgFilterLoad = nil
|
||||
}
|
||||
|
||||
// hash returns the bit offset in the bloom filter which corresponds to the
|
||||
// passed data for the given indepedent hash function number.
|
||||
func (bf *Filter) hash(hashNum uint32, data []byte) uint32 {
|
||||
// bitcoind: 0xfba4c795 chosen as it guarantees a reasonable bit
|
||||
// difference between hashNum values.
|
||||
//
|
||||
// Note that << 3 is equivalent to multiplying by 8, but is faster.
|
||||
// Thus the returned hash is brought into range of the number of bits
|
||||
// the filter has and returned.
|
||||
mm := MurmurHash3(hashNum*0xfba4c795+bf.msgFilterLoad.Tweak, data)
|
||||
return mm % (uint32(len(bf.msgFilterLoad.Filter)) << 3)
|
||||
}
|
||||
|
||||
// matches returns true if the bloom filter might contain the passed data and
|
||||
// false if it definitely does not.
|
||||
//
|
||||
// This function MUST be called with the filter lock held.
|
||||
func (bf *Filter) matches(data []byte) bool {
|
||||
if bf.msgFilterLoad == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// The bloom filter does not contain the data if any of the bit offsets
|
||||
// which result from hashing the data using each independent hash
|
||||
// function are not set. The shifts and masks below are a faster
|
||||
// equivalent of:
|
||||
// arrayIndex := idx / 8 (idx >> 3)
|
||||
// bitOffset := idx % 8 (idx & 7)
|
||||
/// if filter[arrayIndex] & 1<<bitOffset == 0 { ... }
|
||||
for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
|
||||
idx := bf.hash(i, data)
|
||||
if bf.msgFilterLoad.Filter[idx>>3]&(1<<(idx&7)) == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Matches returns true if the bloom filter might contain the passed data and
|
||||
// false if it definitely does not.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) Matches(data []byte) bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.matches(data)
|
||||
}
|
||||
|
||||
// matchesOutPoint returns true if the bloom filter might contain the passed
|
||||
// outpoint and false if it definitely does not.
|
||||
//
|
||||
// This function MUST be called with the filter lock held.
|
||||
func (bf *Filter) matchesOutPoint(outpoint *btcwire.OutPoint) bool {
|
||||
// Serialize
|
||||
var buf [btcwire.HashSize + 4]byte
|
||||
copy(buf[:], outpoint.Hash.Bytes())
|
||||
binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
|
||||
|
||||
return bf.matches(buf[:])
|
||||
}
|
||||
|
||||
// MatchesOutPoint returns true if the bloom filter might contain the passed
|
||||
// outpoint and false if it definitely does not.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) MatchesOutPoint(outpoint *btcwire.OutPoint) bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.matchesOutPoint(outpoint)
|
||||
}
|
||||
|
||||
// add adds the passed byte slice to the bloom filter.
|
||||
//
|
||||
// This function MUST be called with the filter lock held.
|
||||
func (bf *Filter) add(data []byte) {
|
||||
if bf.msgFilterLoad == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Adding data to a bloom filter consists of setting all of the bit
|
||||
// offsets which result from hashing the data using each independent
|
||||
// hash function. The shifts and masks below are a faster equivalent
|
||||
// of:
|
||||
// arrayIndex := idx / 8 (idx >> 3)
|
||||
// bitOffset := idx % 8 (idx & 7)
|
||||
/// filter[arrayIndex] |= 1<<bitOffset
|
||||
for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
|
||||
idx := bf.hash(i, data)
|
||||
bf.msgFilterLoad.Filter[idx>>3] |= (1 << (7 & idx))
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds the passed byte slice to the bloom filter.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) Add(data []byte) {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.add(data)
|
||||
}
|
||||
|
||||
// AddShaHash adds the passed btcwire.ShaHash to the Filter.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) AddShaHash(sha *btcwire.ShaHash) {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.add(sha.Bytes())
|
||||
}
|
||||
|
||||
// addOutPoint adds the passed transaction outpoint to the bloom filter.
|
||||
//
|
||||
// This function MUST be called with the filter lock held.
|
||||
func (bf *Filter) addOutPoint(outpoint *btcwire.OutPoint) {
|
||||
// Serialize
|
||||
var buf [btcwire.HashSize + 4]byte
|
||||
copy(buf[:], outpoint.Hash.Bytes())
|
||||
binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
|
||||
|
||||
bf.add(buf[:])
|
||||
}
|
||||
|
||||
// AddOutPoint adds the passed transaction outpoint to the bloom filter.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) AddOutPoint(outpoint *btcwire.OutPoint) {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.addOutPoint(outpoint)
|
||||
}
|
||||
|
||||
// maybeAddOutpoint potentially adds the passed outpoint to the bloom filter
|
||||
// depending on the bloom update flags and the type of the passed public key
|
||||
// script.
|
||||
//
|
||||
// This function MUST be called with the filter lock held.
|
||||
func (bf *Filter) maybeAddOutpoint(pkScript []byte, outHash *btcwire.ShaHash, outIdx uint32) {
|
||||
switch bf.msgFilterLoad.Flags {
|
||||
case btcwire.BloomUpdateAll:
|
||||
outpoint := btcwire.NewOutPoint(outHash, outIdx)
|
||||
bf.addOutPoint(outpoint)
|
||||
case btcwire.BloomUpdateP2PubkeyOnly:
|
||||
class := btcscript.GetScriptClass(pkScript)
|
||||
if class == btcscript.PubKeyTy || class == btcscript.MultiSigTy {
|
||||
outpoint := btcwire.NewOutPoint(outHash, outIdx)
|
||||
bf.addOutPoint(outpoint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// matchTxAndUpdate returns true if the bloom filter matches data within the
|
||||
// passed transaction, otherwise false is returned. If the filter does match
|
||||
// the passed transaction, it will also update the filter depending on the bloom
|
||||
// update flags set via the loaded filter if needed.
|
||||
//
|
||||
// This function MUST be called with the filter lock held.
|
||||
func (bf *Filter) matchTxAndUpdate(tx *btcutil.Tx) bool {
|
||||
// Check if the filter matches the hash of the transaction.
|
||||
// This is useful for finding transactions when they appear in a block.
|
||||
matched := bf.matches(tx.Sha().Bytes())
|
||||
|
||||
// Check if the filter matches any data elements in the public key
|
||||
// scripts of any of the outputs. When it does, add the outpoint that
|
||||
// matched so transactions which spend from the matched transaction are
|
||||
// also included in the filter. This removes the burden of updating the
|
||||
// filter for this scenario from the client. It is also more efficient
|
||||
// on the network since it avoids the need for another filteradd message
|
||||
// from the client and avoids some potential races that could otherwise
|
||||
// occur.
|
||||
for i, txOut := range tx.MsgTx().TxOut {
|
||||
pushedData, err := btcscript.PushedData(txOut.PkScript)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, data := range pushedData {
|
||||
if !bf.matches(data) {
|
||||
continue
|
||||
}
|
||||
|
||||
matched = true
|
||||
bf.maybeAddOutpoint(txOut.PkScript, tx.Sha(), uint32(i))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing more to do if a match has already been made.
|
||||
if matched {
|
||||
return true
|
||||
}
|
||||
|
||||
// At this point, the transaction and none of the data elements in the
|
||||
// public key scripts of its outputs matched.
|
||||
|
||||
// Check if the filter matches any outpoints this transaction spends or
|
||||
// any any data elements in the signature scripts of any of the inputs.
|
||||
for _, txin := range tx.MsgTx().TxIn {
|
||||
if bf.matchesOutPoint(&txin.PreviousOutpoint) {
|
||||
return true
|
||||
}
|
||||
|
||||
pushedData, err := btcscript.PushedData(txin.SignatureScript)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, data := range pushedData {
|
||||
if bf.matches(data) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// MatchTxAndUpdate returns true if the bloom filter matches data within the
|
||||
// passed transaction, otherwise false is returned. If the filter does match
|
||||
// the passed transaction, it will also update the filter depending on the bloom
|
||||
// update flags set via the loaded filter if needed.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) MatchTxAndUpdate(tx *btcutil.Tx) bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.matchTxAndUpdate(tx)
|
||||
}
|
||||
|
||||
// MsgFilterLoad returns the underlying btcwire.MsgFilterLoad for the bloom
|
||||
// filter.
|
||||
//
|
||||
// This function is safe for concurrent access.
|
||||
func (bf *Filter) MsgFilterLoad() *btcwire.MsgFilterLoad {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.msgFilterLoad
|
||||
}
|
||||
|
|
@ -1,24 +1,46 @@
|
|||
package bloomfilter_test
|
||||
package bloom_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcutil/bloomfilter"
|
||||
"github.com/conformal/btcwire"
|
||||
"testing"
|
||||
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcutil/bloom"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
// TestFilterLarge ensures a maximum sized filter can be created.
|
||||
func TestFilterLarge(t *testing.T) {
|
||||
f := bloomfilter.New(100000000, 0, 0.01, btcwire.BloomUpdateNone)
|
||||
|
||||
f := bloom.NewFilter(100000000, 0, 0.01, btcwire.BloomUpdateNone)
|
||||
if len(f.MsgFilterLoad().Filter) > btcwire.MaxFilterLoadFilterSize {
|
||||
t.Errorf("TestFilterLarge test failed: %d > %d",
|
||||
len(f.MsgFilterLoad().Filter), btcwire.MaxFilterLoadFilterSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterInsert1(t *testing.T) {
|
||||
// TestFilterLoad ensures loading and unloading of a filter pass.
|
||||
func TestFilterLoad(t *testing.T) {
|
||||
merkle := btcwire.MsgFilterLoad{}
|
||||
|
||||
f := bloom.LoadFilter(&merkle)
|
||||
if !f.IsLoaded() {
|
||||
t.Errorf("TestFilterLoad IsLoaded test failed: want %d got %d",
|
||||
true, !f.IsLoaded())
|
||||
return
|
||||
}
|
||||
f.Unload()
|
||||
if f.IsLoaded() {
|
||||
t.Errorf("TestFilterLoad IsLoaded test failed: want %d got %d",
|
||||
f.IsLoaded(), false)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterInsert ensures inserting data into the filter causes that data
|
||||
// to be matched and the resulting serialized MsgFilterLoad is the expected
|
||||
// value.
|
||||
func TestFilterInsert(t *testing.T) {
|
||||
var tests = []struct {
|
||||
hex string
|
||||
insert bool
|
||||
|
|
@ -29,46 +51,49 @@ func TestFilterInsert1(t *testing.T) {
|
|||
{"b9300670b4c5366e95b2699e8b18bc75e5f729c5", true},
|
||||
}
|
||||
|
||||
f := bloomfilter.New(3, 0, 0.01, btcwire.BloomUpdateAll)
|
||||
f := bloom.NewFilter(3, 0, 0.01, btcwire.BloomUpdateAll)
|
||||
|
||||
for x, test := range tests {
|
||||
for i, test := range tests {
|
||||
data, err := hex.DecodeString(test.hex)
|
||||
if err != nil {
|
||||
t.Errorf("TestFilterInsert1 DecodeString failed: %v\n", err)
|
||||
t.Errorf("TestFilterInsert DecodeString failed: %v\n", err)
|
||||
return
|
||||
}
|
||||
if test.insert {
|
||||
f.Add(data)
|
||||
}
|
||||
|
||||
result := f.Contains(data)
|
||||
result := f.Matches(data)
|
||||
if test.insert != result {
|
||||
t.Errorf("TestFilterInsert1 Contains test #%d failure: got %v want %v\n",
|
||||
x, result, test.insert)
|
||||
t.Errorf("TestFilterInsert Matches test #%d failure: got %v want %v\n",
|
||||
i, result, test.insert)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
want, err := hex.DecodeString("03614e9b050000000000000001")
|
||||
if err != nil {
|
||||
t.Errorf("TestFilterInsert1 DecodeString failed: %v\n", err)
|
||||
t.Errorf("TestFilterInsert DecodeString failed: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
got := bytes.NewBuffer(nil)
|
||||
err = f.MsgFilterLoad().BtcEncode(got, btcwire.ProtocolVersion)
|
||||
if err != nil {
|
||||
t.Errorf("TestFilterInsert1 BtcDecode failed: %v\n", err)
|
||||
t.Errorf("TestFilterInsert BtcDecode failed: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !bytes.Equal(got.Bytes(), want) {
|
||||
t.Errorf("TestFilterInsert1 failure: got %v want %v\n",
|
||||
t.Errorf("TestFilterInsert failure: got %v want %v\n",
|
||||
got.Bytes(), want)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterInsert ensures inserting data into the filter with a tweak causes
|
||||
// that data to be matched and the resulting serialized MsgFilterLoad is the
|
||||
// expected value.
|
||||
func TestFilterInsertWithTweak(t *testing.T) {
|
||||
var tests = []struct {
|
||||
hex string
|
||||
|
|
@ -80,9 +105,9 @@ func TestFilterInsertWithTweak(t *testing.T) {
|
|||
{"b9300670b4c5366e95b2699e8b18bc75e5f729c5", true},
|
||||
}
|
||||
|
||||
f := bloomfilter.New(3, 2147483649, 0.01, btcwire.BloomUpdateAll)
|
||||
f := bloom.NewFilter(3, 2147483649, 0.01, btcwire.BloomUpdateAll)
|
||||
|
||||
for x, test := range tests {
|
||||
for i, test := range tests {
|
||||
data, err := hex.DecodeString(test.hex)
|
||||
if err != nil {
|
||||
t.Errorf("TestFilterInsertWithTweak DecodeString failed: %v\n", err)
|
||||
|
|
@ -92,10 +117,10 @@ func TestFilterInsertWithTweak(t *testing.T) {
|
|||
f.Add(data)
|
||||
}
|
||||
|
||||
result := f.Contains(data)
|
||||
result := f.Matches(data)
|
||||
if test.insert != result {
|
||||
t.Errorf("TestFilterInsertWithTweak Contains test #%d failure: got %v want %v\n",
|
||||
x, result, test.insert)
|
||||
t.Errorf("TestFilterInsertWithTweak Matches test #%d failure: got %v want %v\n",
|
||||
i, result, test.insert)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -119,6 +144,8 @@ func TestFilterInsertWithTweak(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestFilterInsertKey ensures inserting public keys and addresses works as
|
||||
// expected.
|
||||
func TestFilterInsertKey(t *testing.T) {
|
||||
secret := "5Kg1gnAjaLfKiwhhPpGS3QfRg2m6awQvaj98JCZBZQ5SuS2F15C"
|
||||
|
||||
|
|
@ -128,8 +155,7 @@ func TestFilterInsertKey(t *testing.T) {
|
|||
return
|
||||
}
|
||||
|
||||
|
||||
f := bloomfilter.New(2, 0, 0.001, btcwire.BloomUpdateAll)
|
||||
f := bloom.NewFilter(2, 0, 0.001, btcwire.BloomUpdateAll)
|
||||
f.Add(wif.SerializePubKey())
|
||||
f.Add(btcutil.Hash160(wif.SerializePubKey()))
|
||||
|
||||
|
|
@ -208,7 +234,7 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
|
||||
f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr := "b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b"
|
||||
sha, err := btcwire.NewShaHashFromStr(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -216,11 +242,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.AddShaHash(sha)
|
||||
if !f.MatchesTx(tx) {
|
||||
if !f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch didn't match sha %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "6bff7fcd4f8565ef406dd5d63d4ff94f318fe82027fd4dc451b04474019f74b4"
|
||||
shaBytes, err := hex.DecodeString(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -228,11 +254,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.Add(shaBytes)
|
||||
if !f.MatchesTx(tx) {
|
||||
if !f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch didn't match sha %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "30450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065" +
|
||||
"f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643" +
|
||||
"ac4cb7cb3c462aced7f14711a01"
|
||||
|
|
@ -242,11 +268,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.Add(shaBytes)
|
||||
if !f.MatchesTx(tx) {
|
||||
if !f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch didn't match input signature %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95" +
|
||||
"c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe" +
|
||||
"76036c339"
|
||||
|
|
@ -256,11 +282,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.Add(shaBytes)
|
||||
if !f.MatchesTx(tx) {
|
||||
if !f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch didn't match input pubkey %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "04943fdd508053c75000106d3bc6e2754dbcff19"
|
||||
shaBytes, err = hex.DecodeString(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -268,14 +294,14 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.Add(shaBytes)
|
||||
if !f.MatchesTx(tx) {
|
||||
if !f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch didn't match output address %s", inputStr)
|
||||
}
|
||||
if !f.MatchesTx(spendingTx) {
|
||||
if !f.MatchTxAndUpdate(spendingTx) {
|
||||
t.Errorf("TestFilterBloomMatch spendingTx didn't match output address %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "a266436d2965547608b9e15d9032a7b9d64fa431"
|
||||
shaBytes, err = hex.DecodeString(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -283,11 +309,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.Add(shaBytes)
|
||||
if !f.MatchesTx(tx) {
|
||||
if !f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch didn't match output address %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"
|
||||
sha, err = btcwire.NewShaHashFromStr(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -296,11 +322,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
}
|
||||
outpoint := btcwire.NewOutPoint(sha, 0)
|
||||
f.AddOutPoint(outpoint)
|
||||
if !f.MatchesTx(tx) {
|
||||
if !f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch didn't match outpoint %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "00000009e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436"
|
||||
sha, err = btcwire.NewShaHashFromStr(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -308,11 +334,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.AddShaHash(sha)
|
||||
if f.MatchesTx(tx) {
|
||||
if f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch matched sha %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "0000006d2965547608b9e15d9032a7b9d64fa431"
|
||||
shaBytes, err = hex.DecodeString(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -320,11 +346,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
return
|
||||
}
|
||||
f.Add(shaBytes)
|
||||
if f.MatchesTx(tx) {
|
||||
if f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch matched address %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"
|
||||
sha, err = btcwire.NewShaHashFromStr(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -333,11 +359,11 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
}
|
||||
outpoint = btcwire.NewOutPoint(sha, 1)
|
||||
f.AddOutPoint(outpoint)
|
||||
if f.MatchesTx(tx) {
|
||||
if f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch matched outpoint %s", inputStr)
|
||||
}
|
||||
|
||||
f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f = bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
inputStr = "000000d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"
|
||||
sha, err = btcwire.NewShaHashFromStr(inputStr)
|
||||
if err != nil {
|
||||
|
|
@ -346,13 +372,13 @@ func TestFilterBloomMatch(t *testing.T) {
|
|||
}
|
||||
outpoint = btcwire.NewOutPoint(sha, 0)
|
||||
f.AddOutPoint(outpoint)
|
||||
if f.MatchesTx(tx) {
|
||||
if f.MatchTxAndUpdate(tx) {
|
||||
t.Errorf("TestFilterBloomMatch matched outpoint %s", inputStr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterInsertUpdateNone(t *testing.T) {
|
||||
f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateNone)
|
||||
f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateNone)
|
||||
|
||||
// Add the generation pubkey
|
||||
inputStr := "04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c" +
|
||||
|
|
@ -382,7 +408,7 @@ func TestFilterInsertUpdateNone(t *testing.T) {
|
|||
}
|
||||
outpoint := btcwire.NewOutPoint(sha, 0)
|
||||
|
||||
if f.ContainsOutPoint(outpoint) {
|
||||
if f.MatchesOutPoint(outpoint) {
|
||||
t.Errorf("TestFilterInsertUpdateNone matched outpoint %s", inputStr)
|
||||
return
|
||||
}
|
||||
|
|
@ -395,7 +421,7 @@ func TestFilterInsertUpdateNone(t *testing.T) {
|
|||
}
|
||||
outpoint = btcwire.NewOutPoint(sha, 0)
|
||||
|
||||
if f.ContainsOutPoint(outpoint) {
|
||||
if f.MatchesOutPoint(outpoint) {
|
||||
t.Errorf("TestFilterInsertUpdateNone matched outpoint %s", inputStr)
|
||||
return
|
||||
}
|
||||
|
|
@ -500,7 +526,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
|
|||
return
|
||||
}
|
||||
|
||||
f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateP2PubkeyOnly)
|
||||
f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateP2PubkeyOnly)
|
||||
|
||||
// Generation pubkey
|
||||
inputStr := "04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c" +
|
||||
|
|
@ -523,7 +549,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
|
|||
f.Add(inputBytes)
|
||||
|
||||
// Ignore return value -- this is just used to update the filter.
|
||||
_, _ = bloomfilter.NewMerkleBlock(block, f)
|
||||
_, _ = bloom.NewMerkleBlock(block, f)
|
||||
|
||||
// We should match the generation pubkey
|
||||
inputStr = "147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"
|
||||
|
|
@ -533,7 +559,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
|
|||
return
|
||||
}
|
||||
outpoint := btcwire.NewOutPoint(sha, 0)
|
||||
if !f.ContainsOutPoint(outpoint) {
|
||||
if !f.MatchesOutPoint(outpoint) {
|
||||
t.Errorf("TestMerkleBlockP2PubKeyOnly didn't match the generation "+
|
||||
"outpoint %s", inputStr)
|
||||
return
|
||||
|
|
@ -547,7 +573,7 @@ func TestFilterInsertP2PubKeyOnly(t *testing.T) {
|
|||
return
|
||||
}
|
||||
outpoint = btcwire.NewOutPoint(sha, 0)
|
||||
if f.ContainsOutPoint(outpoint) {
|
||||
if f.MatchesOutPoint(outpoint) {
|
||||
t.Errorf("TestMerkleBlockP2PubKeyOnly matched outpoint %s", inputStr)
|
||||
return
|
||||
}
|
||||
120
bloom/merkleblock.go
Normal file
120
bloom/merkleblock.go
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
package bloom
|
||||
|
||||
import (
|
||||
"github.com/conformal/btcchain"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
// merkleBlock is used to house intermediate information needed to generate a
|
||||
// btcwire.MsgMerkleBlock according to a filter.
|
||||
type merkleBlock struct {
|
||||
numTx uint32
|
||||
allHashes []*btcwire.ShaHash
|
||||
finalHashes []*btcwire.ShaHash
|
||||
matchedBits []byte
|
||||
bits []byte
|
||||
}
|
||||
|
||||
// calcTreeWidth calculates and returns the the number of nodes (width) or a
|
||||
// merkle tree at the given depth-first height.
|
||||
func (m *merkleBlock) calcTreeWidth(height uint32) uint32 {
|
||||
return (m.numTx + (1 << height) - 1) >> height
|
||||
}
|
||||
|
||||
// calcHash returns the hash for a sub-tree given a depth-first height and
|
||||
// node position.
|
||||
func (m *merkleBlock) calcHash(height, pos uint32) *btcwire.ShaHash {
|
||||
if height == 0 {
|
||||
return m.allHashes[pos]
|
||||
}
|
||||
|
||||
var right *btcwire.ShaHash
|
||||
left := m.calcHash(height-1, pos*2)
|
||||
if pos*2+1 < m.calcTreeWidth(height-1) {
|
||||
right = m.calcHash(height-1, pos*2+1)
|
||||
} else {
|
||||
right = left
|
||||
}
|
||||
return btcchain.HashMerkleBranches(left, right)
|
||||
}
|
||||
|
||||
// traverseAndBuild builds a partial merkle tree using a recursive depth-first
|
||||
// approach. As it calculates the hashes, it also saves whether or not each
|
||||
// node is a parent node and a list of final hashes to be included in the
|
||||
// merkle block.
|
||||
func (m *merkleBlock) traverseAndBuild(height, pos uint32) {
|
||||
// Determine whether this node is a parent of a matched node.
|
||||
var isParent byte
|
||||
for i := pos << height; i < (pos+1)<<height && i < m.numTx; i++ {
|
||||
isParent |= m.matchedBits[i]
|
||||
}
|
||||
m.bits = append(m.bits, isParent)
|
||||
|
||||
// When the node is a leaf node or not a parent of a matched node,
|
||||
// append the hash to the list that will be part of the final merkle
|
||||
// block.
|
||||
if height == 0 || isParent == 0x00 {
|
||||
m.finalHashes = append(m.finalHashes, m.calcHash(height, pos))
|
||||
return
|
||||
}
|
||||
|
||||
// At this point, the node is an internal node and it is the parent of
|
||||
// of an included leaf node.
|
||||
|
||||
// Descend into the left child and process its sub-tree.
|
||||
m.traverseAndBuild(height-1, pos*2)
|
||||
|
||||
// Descend into the right child and process its sub-tree if
|
||||
// there is one.
|
||||
if pos*2+1 < m.calcTreeWidth(height-1) {
|
||||
m.traverseAndBuild(height-1, pos*2+1)
|
||||
}
|
||||
}
|
||||
|
||||
// NewMerkleBlock returns a new *btcwire.MsgMerkleBlock based on the passed
|
||||
// block and filter.
|
||||
func NewMerkleBlock(block *btcutil.Block, filter *Filter) (*btcwire.MsgMerkleBlock, []*btcwire.ShaHash) {
|
||||
numTx := uint32(len(block.Transactions()))
|
||||
mBlock := merkleBlock{
|
||||
numTx: numTx,
|
||||
allHashes: make([]*btcwire.ShaHash, 0, numTx),
|
||||
matchedBits: make([]byte, 0, numTx),
|
||||
}
|
||||
|
||||
// Find and keep track of any transactions that match the filter.
|
||||
var matchedHashes []*btcwire.ShaHash
|
||||
for _, tx := range block.Transactions() {
|
||||
if filter.MatchTxAndUpdate(tx) {
|
||||
mBlock.matchedBits = append(mBlock.matchedBits, 0x01)
|
||||
matchedHashes = append(matchedHashes, tx.Sha())
|
||||
} else {
|
||||
mBlock.matchedBits = append(mBlock.matchedBits, 0x00)
|
||||
}
|
||||
mBlock.allHashes = append(mBlock.allHashes, tx.Sha())
|
||||
}
|
||||
|
||||
// Calculate the number of merkle branches (height) in the tree.
|
||||
height := uint32(0)
|
||||
for mBlock.calcTreeWidth(height) > 1 {
|
||||
height++
|
||||
}
|
||||
|
||||
// Build the depth-first partial merkle tree.
|
||||
mBlock.traverseAndBuild(height, 0)
|
||||
|
||||
// Create and return the merkle block.
|
||||
msgMerkleBlock := btcwire.MsgMerkleBlock{
|
||||
Header: block.MsgBlock().Header,
|
||||
Transactions: uint32(mBlock.numTx),
|
||||
Hashes: make([]*btcwire.ShaHash, 0, len(mBlock.finalHashes)),
|
||||
Flags: make([]byte, (len(mBlock.bits)+7)/8),
|
||||
}
|
||||
for _, sha := range mBlock.finalHashes {
|
||||
msgMerkleBlock.AddTxHash(sha)
|
||||
}
|
||||
for i := uint32(0); i < uint32(len(mBlock.bits)); i++ {
|
||||
msgMerkleBlock.Flags[i/8] |= mBlock.bits[i] << (i % 8)
|
||||
}
|
||||
return &msgMerkleBlock, matchedHashes
|
||||
}
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
package bloomfilter_test
|
||||
package bloom_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcutil/bloomfilter"
|
||||
"github.com/conformal/btcwire"
|
||||
"testing"
|
||||
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcutil/bloom"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
func TestMerkleBlock3(t *testing.T) {
|
||||
|
|
@ -29,7 +30,7 @@ func TestMerkleBlock3(t *testing.T) {
|
|||
return
|
||||
}
|
||||
|
||||
f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
f := bloom.NewFilter(10, 0, 0.000001, btcwire.BloomUpdateAll)
|
||||
|
||||
inputStr := "63194f18be0af63f2c6bc9dc0f777cbefed3d9415c4af83f3ee3a3d669c00cb5"
|
||||
sha, err := btcwire.NewShaHashFromStr(inputStr)
|
||||
|
|
@ -40,7 +41,7 @@ func TestMerkleBlock3(t *testing.T) {
|
|||
|
||||
f.AddShaHash(sha)
|
||||
|
||||
mBlock, _ := bloomfilter.NewMerkleBlock(blk, f)
|
||||
mBlock, _ := bloom.NewMerkleBlock(blk, f)
|
||||
|
||||
wantStr := "0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4" +
|
||||
"b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc" +
|
||||
68
bloom/murmurhash3.go
Normal file
68
bloom/murmurhash3.go
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
package bloom
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// The following constants are used by the MurmurHash3 algorithm.
|
||||
const (
|
||||
murmurC1 = 0xcc9e2d51
|
||||
murmurC2 = 0x1b873593
|
||||
murmurR1 = 15
|
||||
murmurR2 = 13
|
||||
murmurM = 5
|
||||
murmurN = 0xe6546b64
|
||||
)
|
||||
|
||||
// MurmurHash3 implements a non-cryptographic hash function using the
|
||||
// MurmurHash3 algorithm. This implementation yields a 32-bit hash value which
|
||||
// is suitable for general hash-based lookups. The seed can be used to
|
||||
// effectively randomize the hash function. This makes it ideal for use in
|
||||
// bloom filters which need multiple independent hash functions.
|
||||
func MurmurHash3(seed uint32, data []byte) uint32 {
|
||||
dataLen := uint32(len(data))
|
||||
hash := seed
|
||||
k := uint32(0)
|
||||
numBlocks := dataLen / 4
|
||||
|
||||
// Calculate the hash in 4-byte chunks.
|
||||
for i := uint32(0); i < numBlocks; i++ {
|
||||
k = binary.LittleEndian.Uint32(data[i*4:])
|
||||
k *= murmurC1
|
||||
k = (k << murmurR1) | (k >> (32 - murmurR1))
|
||||
k *= murmurC2
|
||||
|
||||
hash ^= k
|
||||
hash = (hash << murmurR2) | (hash >> (32 - murmurR2))
|
||||
hash = hash*murmurM + murmurN
|
||||
}
|
||||
|
||||
// Handle remaining bytes.
|
||||
tailIdx := numBlocks * 4
|
||||
k = 0
|
||||
|
||||
switch dataLen & 3 {
|
||||
case 3:
|
||||
k ^= uint32(data[tailIdx+2]) << 16
|
||||
fallthrough
|
||||
case 2:
|
||||
k ^= uint32(data[tailIdx+1]) << 8
|
||||
fallthrough
|
||||
case 1:
|
||||
k ^= uint32(data[tailIdx])
|
||||
k *= murmurC1
|
||||
k = (k << murmurR1) | (k >> (32 - murmurR1))
|
||||
k *= murmurC2
|
||||
hash ^= k
|
||||
}
|
||||
|
||||
// Finalization.
|
||||
hash ^= uint32(dataLen)
|
||||
hash ^= hash >> 16
|
||||
hash *= 0x85ebca6b
|
||||
hash ^= hash >> 13
|
||||
hash *= 0xc2b2ae35
|
||||
hash ^= hash >> 16
|
||||
|
||||
return hash
|
||||
}
|
||||
|
|
@ -1,10 +1,13 @@
|
|||
package bloomfilter_test
|
||||
package bloom_test
|
||||
|
||||
import (
|
||||
"github.com/conformal/btcutil/bloomfilter"
|
||||
"testing"
|
||||
|
||||
"github.com/conformal/btcutil/bloom"
|
||||
)
|
||||
|
||||
// TestMurmurHash3 ensure the MurmurHash3 function produces the correct hash
|
||||
// when given various seeds and data.
|
||||
func TestMurmurHash3(t *testing.T) {
|
||||
var tests = []struct {
|
||||
seed uint32
|
||||
|
|
@ -27,11 +30,11 @@ func TestMurmurHash3(t *testing.T) {
|
|||
{0x00000000, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88}, 0xb4698def},
|
||||
}
|
||||
|
||||
for x, test := range tests {
|
||||
result := bloomfilter.MurmurHash3(test.seed, test.data)
|
||||
for i, test := range tests {
|
||||
result := bloom.MurmurHash3(test.seed, test.data)
|
||||
if result != test.out {
|
||||
t.Errorf("MurmurHash3 test #%d failed: got %v want %v\n",
|
||||
x, result, test.out)
|
||||
i, result, test.out)
|
||||
continue
|
||||
}
|
||||
}
|
||||
27
bloom/test_coverage.txt
Normal file
27
bloom/test_coverage.txt
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
|
||||
github.com/conformal/btcutil/bloom/murmurhash3.go MurmurHash3 100.00% (31/31)
|
||||
github.com/conformal/btcutil/bloom/merkleblock.go NewMerkleBlock 100.00% (22/22)
|
||||
github.com/conformal/btcutil/bloom/merkleblock.go merkleBlock.traverseAndBuild 100.00% (9/9)
|
||||
github.com/conformal/btcutil/bloom/merkleblock.go merkleBlock.calcHash 100.00% (8/8)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.maybeAddOutpoint 100.00% (7/7)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.matchesOutPoint 100.00% (4/4)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.addOutPoint 100.00% (4/4)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.AddShaHash 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.IsLoaded 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.Unload 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.Matches 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.MatchesOutPoint 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.Add 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.AddOutPoint 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.MatchTxAndUpdate 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.MsgFilterLoad 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go minUint32 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.hash 100.00% (2/2)
|
||||
github.com/conformal/btcutil/bloom/filter.go LoadFilter 100.00% (1/1)
|
||||
github.com/conformal/btcutil/bloom/merkleblock.go merkleBlock.calcTreeWidth 100.00% (1/1)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.matchTxAndUpdate 91.30% (21/23)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.matches 85.71% (6/7)
|
||||
github.com/conformal/btcutil/bloom/filter.go NewFilter 81.82% (9/11)
|
||||
github.com/conformal/btcutil/bloom/filter.go Filter.add 80.00% (4/5)
|
||||
github.com/conformal/btcutil/bloom ---------------------------- 96.36% (159/165)
|
||||
|
||||
|
|
@ -1,225 +0,0 @@
|
|||
// Copyright (c) 2014 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package bloomfilter
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"github.com/conformal/btcscript"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
"math"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// BloomFilter defines a bitcoin bloomfilter the provides easy manipulation of raw
|
||||
// filter data.
|
||||
type BloomFilter struct {
|
||||
sync.Mutex
|
||||
msgFilterLoad *btcwire.MsgFilterLoad
|
||||
}
|
||||
|
||||
// New creates a new Filter instance, mainly to be used by SPV clients. The tweak parameter is
|
||||
// a random value added to the seed value. For more information on what values to use for both
|
||||
// elements and fprate, please see https://en.wikipedia.org/wiki/Bloom_filter.
|
||||
func New(elements, tweak uint32, fprate float64, flags btcwire.BloomUpdateType) *BloomFilter {
|
||||
dataLen := uint32(math.Abs(math.Log(fprate)) * float64(elements) / (math.Ln2 * math.Ln2))
|
||||
dataLen = min(dataLen, btcwire.MaxFilterLoadFilterSize*8) / 8
|
||||
|
||||
hashFuncs := min(uint32(float64(dataLen)*8.0/float64(elements)*math.Ln2), btcwire.MaxFilterLoadHashFuncs)
|
||||
|
||||
data := make([]byte, dataLen)
|
||||
msg := btcwire.NewMsgFilterLoad(data, hashFuncs, tweak, flags)
|
||||
|
||||
return &BloomFilter{
|
||||
msgFilterLoad: msg,
|
||||
}
|
||||
}
|
||||
|
||||
// Load creates a new BloomFilter instance with the given btcwire.MsgFilterLoad.
|
||||
func Load(filter *btcwire.MsgFilterLoad) *BloomFilter {
|
||||
return &BloomFilter{
|
||||
msgFilterLoad: filter,
|
||||
}
|
||||
}
|
||||
|
||||
// Loaded returns true if a filter is loaded, otherwise false.
|
||||
func (bf *BloomFilter) IsLoaded() bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.msgFilterLoad != nil
|
||||
}
|
||||
|
||||
// Unload clears the Filter.
|
||||
func (bf *BloomFilter) Unload() {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.msgFilterLoad = nil
|
||||
}
|
||||
|
||||
func (bf *BloomFilter) contains(data []byte) bool {
|
||||
if bf.msgFilterLoad == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
|
||||
idx := bf.hash(i, data)
|
||||
if bf.msgFilterLoad.Filter[idx>>3]&(1<<(7&idx)) != 1<<(7&idx) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Contains returns true if the BloomFilter contains the passed byte slice. Otherwise,
|
||||
// it returns false.
|
||||
func (bf *BloomFilter) Contains(data []byte) bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.contains(data)
|
||||
}
|
||||
|
||||
func (bf *BloomFilter) containsOutPoint(outpoint *btcwire.OutPoint) bool {
|
||||
// Serialize
|
||||
var buf [btcwire.HashSize + 4]byte
|
||||
copy(buf[:], outpoint.Hash.Bytes())
|
||||
binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
|
||||
|
||||
return bf.contains(buf[:])
|
||||
}
|
||||
|
||||
// ContainsOutPoint returns true if the BloomFilter contains the given
|
||||
// btcwire.OutPoint. Otherwise, it returns false.
|
||||
func (bf *BloomFilter) ContainsOutPoint(outpoint *btcwire.OutPoint) bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.containsOutPoint(outpoint)
|
||||
}
|
||||
|
||||
func (bf *BloomFilter) add(data []byte) {
|
||||
if bf.msgFilterLoad == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
|
||||
idx := bf.hash(i, data)
|
||||
bf.msgFilterLoad.Filter[idx>>3] |= (1 << (7 & idx))
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds the passed byte slice to the BloomFilter.
|
||||
func (bf *BloomFilter) Add(data []byte) {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.add(data)
|
||||
}
|
||||
|
||||
// AddShaHash adds the passed btcwire.ShaHash to the BloomFilter.
|
||||
func (bf *BloomFilter) AddShaHash(sha *btcwire.ShaHash) {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.add(sha.Bytes())
|
||||
}
|
||||
|
||||
// AddOutPoint adds the passed btcwire.OutPoint to the BloomFilter.
|
||||
func (bf *BloomFilter) AddOutPoint(outpoint *btcwire.OutPoint) {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
bf.addOutPoint(outpoint)
|
||||
}
|
||||
|
||||
func (bf *BloomFilter) addOutPoint(outpoint *btcwire.OutPoint) {
|
||||
// Serialize
|
||||
var buf [btcwire.HashSize + 4]byte
|
||||
copy(buf[:], outpoint.Hash.Bytes())
|
||||
binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index)
|
||||
|
||||
bf.add(buf[:])
|
||||
}
|
||||
|
||||
func (bf *BloomFilter) matches(tx *btcutil.Tx) bool {
|
||||
hash := tx.Sha().Bytes()
|
||||
matched := bf.contains(hash)
|
||||
|
||||
for i, txout := range tx.MsgTx().TxOut {
|
||||
pushedData, err := btcscript.PushedData(txout.PkScript)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, p := range pushedData {
|
||||
if bf.contains(p) {
|
||||
switch bf.msgFilterLoad.Flags {
|
||||
case btcwire.BloomUpdateAll:
|
||||
outpoint := btcwire.NewOutPoint(tx.Sha(), uint32(i))
|
||||
bf.addOutPoint(outpoint)
|
||||
case btcwire.BloomUpdateP2PubkeyOnly:
|
||||
class := btcscript.GetScriptClass(txout.PkScript)
|
||||
if class == btcscript.PubKeyTy || class == btcscript.MultiSigTy {
|
||||
outpoint := btcwire.NewOutPoint(tx.Sha(), uint32(i))
|
||||
bf.addOutPoint(outpoint)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if matched {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, txin := range tx.MsgTx().TxIn {
|
||||
if bf.containsOutPoint(&txin.PreviousOutpoint) {
|
||||
return true
|
||||
}
|
||||
pushedData, err := btcscript.PushedData(txin.SignatureScript)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, p := range pushedData {
|
||||
if bf.contains(p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MatchesTx returns true if the BloomFilter matches data within the passed transaction,
|
||||
// otherwise false is returned. If the BloomFilter does match the passed transaction,
|
||||
// it will also update the BloomFilter if required.
|
||||
func (bf *BloomFilter) MatchesTx(tx *btcutil.Tx) bool {
|
||||
bf.Lock()
|
||||
defer bf.Unlock()
|
||||
|
||||
return bf.matches(tx)
|
||||
}
|
||||
|
||||
// MsgFilterLoad returns the underlying btcwire.MsgFilterLoad for the BloomFilter.
|
||||
func (bf *BloomFilter) MsgFilterLoad() *btcwire.MsgFilterLoad {
|
||||
return bf.msgFilterLoad
|
||||
}
|
||||
|
||||
func (bf *BloomFilter) hash(hashNum uint32, data []byte) uint32 {
|
||||
// bitcoind: 0xFBA4C795 chosen as it guarantees a reasonable bit
|
||||
// difference between nHashNum values.
|
||||
mm := MurmurHash3(hashNum*0xFBA4C795+bf.msgFilterLoad.Tweak, data)
|
||||
return mm % (uint32(len(bf.msgFilterLoad.Filter)) * 8)
|
||||
}
|
||||
|
||||
// min is a convenience function to return the minimum value of the two
|
||||
// passed uint32 values.
|
||||
func min(a, b uint32) uint32 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
package bloomfilter
|
||||
|
||||
import (
|
||||
"github.com/conformal/btcchain"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
type merkleBlock struct {
|
||||
msgMerkleBlock *btcwire.MsgMerkleBlock
|
||||
allHashes []*btcwire.ShaHash
|
||||
finalHashes []*btcwire.ShaHash
|
||||
matchedBits []byte
|
||||
bits []byte
|
||||
}
|
||||
|
||||
// NewMerkleBlock returns a new *btcwire.MsgMerkleBlock based on the passed
|
||||
// block and filter.
|
||||
func NewMerkleBlock(block *btcutil.Block, filter *BloomFilter) (*btcwire.MsgMerkleBlock, []*btcwire.ShaHash) {
|
||||
blockHeader := block.MsgBlock().Header
|
||||
mBlock := merkleBlock{
|
||||
msgMerkleBlock: btcwire.NewMsgMerkleBlock(&blockHeader),
|
||||
}
|
||||
|
||||
var matchedHashes []*btcwire.ShaHash
|
||||
for _, tx := range block.Transactions() {
|
||||
if filter.MatchesTx(tx) {
|
||||
mBlock.matchedBits = append(mBlock.matchedBits, 0x01)
|
||||
matchedHashes = append(matchedHashes, tx.Sha())
|
||||
} else {
|
||||
mBlock.matchedBits = append(mBlock.matchedBits, 0x00)
|
||||
}
|
||||
mBlock.allHashes = append(mBlock.allHashes, tx.Sha())
|
||||
}
|
||||
mBlock.msgMerkleBlock.Transactions = uint32(len(block.Transactions()))
|
||||
|
||||
height := uint32(0)
|
||||
for mBlock.calcTreeWidth(height) > 1 {
|
||||
height++
|
||||
}
|
||||
|
||||
mBlock.traverseAndBuild(height, 0)
|
||||
|
||||
for _, sha := range mBlock.finalHashes {
|
||||
mBlock.msgMerkleBlock.AddTxHash(sha)
|
||||
}
|
||||
mBlock.msgMerkleBlock.Flags = make([]byte, (len(mBlock.bits)+7)/8)
|
||||
for i := uint32(0); i < uint32(len(mBlock.bits)); i++ {
|
||||
mBlock.msgMerkleBlock.Flags[i/8] |= mBlock.bits[i] << (i % 8)
|
||||
}
|
||||
return mBlock.msgMerkleBlock, matchedHashes
|
||||
}
|
||||
|
||||
func (m *merkleBlock) calcTreeWidth(height uint32) uint32 {
|
||||
return (m.msgMerkleBlock.Transactions + (1 << height) - 1) >> height
|
||||
}
|
||||
|
||||
func (m *merkleBlock) calcHash(height, pos uint32) *btcwire.ShaHash {
|
||||
if height == 0 {
|
||||
return m.allHashes[pos]
|
||||
} else {
|
||||
var right *btcwire.ShaHash
|
||||
left := m.calcHash(height-1, pos*2)
|
||||
if pos*2+1 < m.calcTreeWidth(height-1) {
|
||||
right = m.calcHash(height-1, pos*2+1)
|
||||
} else {
|
||||
right = left
|
||||
}
|
||||
return btcchain.HashMerkleBranches(left, right)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *merkleBlock) traverseAndBuild(height, pos uint32) {
|
||||
var isParent byte
|
||||
|
||||
for i := pos << height; i < (pos+1)<<height && i < m.msgMerkleBlock.Transactions; i++ {
|
||||
isParent |= m.matchedBits[i]
|
||||
}
|
||||
|
||||
m.bits = append(m.bits, isParent)
|
||||
if height == 0 || isParent == 0x00 {
|
||||
m.finalHashes = append(m.finalHashes, m.calcHash(height, pos))
|
||||
} else {
|
||||
m.traverseAndBuild(height-1, pos*2)
|
||||
if pos*2+1 < m.calcTreeWidth(height-1) {
|
||||
m.traverseAndBuild(height-1, pos*2+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
package bloomfilter
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// MurmurHash3 implements the MurmurHash3 algorithm.
|
||||
func MurmurHash3(seed uint32, data []byte) uint32 {
|
||||
dataLen := len(data)
|
||||
h1 := seed
|
||||
c1 := uint32(0xcc9e2d51)
|
||||
c2 := uint32(0x1b873593)
|
||||
k1 := uint32(0)
|
||||
numBlocks := dataLen / 4
|
||||
|
||||
// body
|
||||
for i := 0; i < numBlocks; i++ {
|
||||
k1 = binary.LittleEndian.Uint32(data[i * 4:])
|
||||
k1 *= c1
|
||||
k1 = (k1 << 15) | (k1 >> (32 - 15))
|
||||
k1 *= c2
|
||||
h1 ^= k1
|
||||
h1 = (h1 << 13) | (h1 >> (32 - 13))
|
||||
h1 = h1*5 + 0xe6546b64
|
||||
}
|
||||
|
||||
// tail
|
||||
tailidx := numBlocks * 4
|
||||
k1 = 0
|
||||
|
||||
switch dataLen & 3 {
|
||||
case 3:
|
||||
k1 ^= uint32(data[tailidx+2]) << 16
|
||||
fallthrough
|
||||
case 2:
|
||||
k1 ^= uint32(data[tailidx+1]) << 8
|
||||
fallthrough
|
||||
case 1:
|
||||
k1 ^= uint32(data[tailidx])
|
||||
k1 *= c1
|
||||
k1 = (k1 << 15) | (k1 >> (32 - 15))
|
||||
k1 *= c2
|
||||
h1 ^= k1
|
||||
}
|
||||
|
||||
// Finalization
|
||||
h1 ^= uint32(dataLen)
|
||||
h1 ^= h1 >> 16
|
||||
h1 *= 0x85ebca6b
|
||||
h1 ^= h1 >> 13
|
||||
h1 *= 0xc2b2ae35
|
||||
h1 ^= h1 >> 16
|
||||
|
||||
return h1
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
|
||||
github.com/conformal/btcutil/bloomfilter/murmurhash3.go MurmurHash3 100.00% (34/34)
|
||||
github.com/conformal/btcutil/bloomfilter/merkleblock.go NewMerkleBlock 100.00% (20/20)
|
||||
github.com/conformal/btcutil/bloomfilter/merkleblock.go merkleBlock.traverseAndBuild 100.00% (9/9)
|
||||
github.com/conformal/btcutil/bloomfilter/merkleblock.go merkleBlock.calcHash 100.00% (8/8)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go New 100.00% (6/6)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.containsOutPoint 100.00% (4/4)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.addOutPoint 100.00% (4/4)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.Add 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.AddShaHash 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.Contains 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.ContainsOutPoint 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.MatchesTx 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go min 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.AddOutPoint 100.00% (3/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.hash 100.00% (2/2)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.MsgFilterLoad 100.00% (1/1)
|
||||
github.com/conformal/btcutil/bloomfilter/merkleblock.go merkleBlock.calcTreeWidth 100.00% (1/1)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.matches 92.86% (26/28)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.contains 85.71% (6/7)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.add 80.00% (4/5)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.Unload 0.00% (0/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.IsLoaded 0.00% (0/3)
|
||||
github.com/conformal/btcutil/bloomfilter/filter.go Load 0.00% (0/1)
|
||||
github.com/conformal/btcutil/bloomfilter ---------------------------- 92.99% (146/157)
|
||||
|
||||
Loading…
Reference in a new issue