From 9e3269e67ca85885b06528ac5bdfad8f7f0b7620 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 30 Jun 2014 20:51:45 -0400 Subject: [PATCH] Initial work towards BIP0037 bloom filtering API. --- bloomfilter/cov_report.sh | 17 + bloomfilter/filter.go | 225 +++++++++++++ bloomfilter/filter_test.go | 554 ++++++++++++++++++++++++++++++++ bloomfilter/merkleblock.go | 89 +++++ bloomfilter/merkleblock_test.go | 68 ++++ bloomfilter/murmurhash3.go | 55 ++++ bloomfilter/murmurhash3_test.go | 38 +++ bloomfilter/test_coverage.txt | 26 ++ 8 files changed, 1072 insertions(+) create mode 100644 bloomfilter/cov_report.sh create mode 100644 bloomfilter/filter.go create mode 100644 bloomfilter/filter_test.go create mode 100644 bloomfilter/merkleblock.go create mode 100644 bloomfilter/merkleblock_test.go create mode 100644 bloomfilter/murmurhash3.go create mode 100644 bloomfilter/murmurhash3_test.go create mode 100644 bloomfilter/test_coverage.txt diff --git a/bloomfilter/cov_report.sh b/bloomfilter/cov_report.sh new file mode 100644 index 0000000..307f05b --- /dev/null +++ b/bloomfilter/cov_report.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# This script uses gocov to generate a test coverage report. +# The gocov tool my be obtained with the following command: +# go get github.com/axw/gocov/gocov +# +# It will be installed to $GOPATH/bin, so ensure that location is in your $PATH. + +# Check for gocov. +type gocov >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echo >&2 "This script requires the gocov tool." + echo >&2 "You may obtain it with the following command:" + echo >&2 "go get github.com/axw/gocov/gocov" + exit 1 +fi +gocov test | gocov report diff --git a/bloomfilter/filter.go b/bloomfilter/filter.go new file mode 100644 index 0000000..af1b128 --- /dev/null +++ b/bloomfilter/filter.go @@ -0,0 +1,225 @@ +// Copyright (c) 2014 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package bloomfilter + +import ( + "encoding/binary" + "github.com/conformal/btcscript" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "math" + "sync" +) + +// BloomFilter defines a bitcoin bloomfilter the provides easy manipulation of raw +// filter data. +type BloomFilter struct { + sync.Mutex + msgFilterLoad *btcwire.MsgFilterLoad +} + +// New creates a new Filter instance, mainly to be used by SPV clients. The tweak parameter is +// a random value added to the seed value. For more information on what values to use for both +// elements and fprate, please see https://en.wikipedia.org/wiki/Bloom_filter. +func New(elements, tweak uint32, fprate float64, flags btcwire.BloomUpdateType) *BloomFilter { + dataLen := uint32(math.Abs(math.Log(fprate)) * float64(elements) / (math.Ln2 * math.Ln2)) + dataLen = min(dataLen, btcwire.MaxFilterLoadFilterSize*8) / 8 + + hashFuncs := min(uint32(float64(dataLen)*8.0/float64(elements)*math.Ln2), btcwire.MaxFilterLoadHashFuncs) + + data := make([]byte, dataLen) + msg := btcwire.NewMsgFilterLoad(data, hashFuncs, tweak, flags) + + return &BloomFilter{ + msgFilterLoad: msg, + } +} + +// Load creates a new BloomFilter instance with the given btcwire.MsgFilterLoad. +func Load(filter *btcwire.MsgFilterLoad) *BloomFilter { + return &BloomFilter{ + msgFilterLoad: filter, + } +} + +// Loaded returns true if a filter is loaded, otherwise false. +func (bf *BloomFilter) IsLoaded() bool { + bf.Lock() + defer bf.Unlock() + + return bf.msgFilterLoad != nil +} + +// Unload clears the Filter. +func (bf *BloomFilter) Unload() { + bf.Lock() + defer bf.Unlock() + + bf.msgFilterLoad = nil +} + +func (bf *BloomFilter) contains(data []byte) bool { + if bf.msgFilterLoad == nil { + return false + } + + for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ { + idx := bf.hash(i, data) + if bf.msgFilterLoad.Filter[idx>>3]&(1<<(7&idx)) != 1<<(7&idx) { + return false + } + } + return true +} + +// Contains returns true if the BloomFilter contains the passed byte slice. Otherwise, +// it returns false. +func (bf *BloomFilter) Contains(data []byte) bool { + bf.Lock() + defer bf.Unlock() + + return bf.contains(data) +} + +func (bf *BloomFilter) containsOutPoint(outpoint *btcwire.OutPoint) bool { + // Serialize + var buf [btcwire.HashSize + 4]byte + copy(buf[:], outpoint.Hash.Bytes()) + binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index) + + return bf.contains(buf[:]) +} + +// ContainsOutPoint returns true if the BloomFilter contains the given +// btcwire.OutPoint. Otherwise, it returns false. +func (bf *BloomFilter) ContainsOutPoint(outpoint *btcwire.OutPoint) bool { + bf.Lock() + defer bf.Unlock() + + return bf.containsOutPoint(outpoint) +} + +func (bf *BloomFilter) add(data []byte) { + if bf.msgFilterLoad == nil { + return + } + + for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ { + idx := bf.hash(i, data) + bf.msgFilterLoad.Filter[idx>>3] |= (1 << (7 & idx)) + } +} + +// Add adds the passed byte slice to the BloomFilter. +func (bf *BloomFilter) Add(data []byte) { + bf.Lock() + defer bf.Unlock() + + bf.add(data) +} + +// AddShaHash adds the passed btcwire.ShaHash to the BloomFilter. +func (bf *BloomFilter) AddShaHash(sha *btcwire.ShaHash) { + bf.Lock() + defer bf.Unlock() + + bf.add(sha.Bytes()) +} + +// AddOutPoint adds the passed btcwire.OutPoint to the BloomFilter. +func (bf *BloomFilter) AddOutPoint(outpoint *btcwire.OutPoint) { + bf.Lock() + defer bf.Unlock() + + bf.addOutPoint(outpoint) +} + +func (bf *BloomFilter) addOutPoint(outpoint *btcwire.OutPoint) { + // Serialize + var buf [btcwire.HashSize + 4]byte + copy(buf[:], outpoint.Hash.Bytes()) + binary.LittleEndian.PutUint32(buf[btcwire.HashSize:], outpoint.Index) + + bf.add(buf[:]) +} + +func (bf *BloomFilter) matches(tx *btcutil.Tx) bool { + hash := tx.Sha().Bytes() + matched := bf.contains(hash) + + for i, txout := range tx.MsgTx().TxOut { + pushedData, err := btcscript.PushedData(txout.PkScript) + if err != nil { + break + } + for _, p := range pushedData { + if bf.contains(p) { + switch bf.msgFilterLoad.Flags { + case btcwire.BloomUpdateAll: + outpoint := btcwire.NewOutPoint(tx.Sha(), uint32(i)) + bf.addOutPoint(outpoint) + case btcwire.BloomUpdateP2PubkeyOnly: + class := btcscript.GetScriptClass(txout.PkScript) + if class == btcscript.PubKeyTy || class == btcscript.MultiSigTy { + outpoint := btcwire.NewOutPoint(tx.Sha(), uint32(i)) + bf.addOutPoint(outpoint) + } + } + return true + } + } + } + + if matched { + return true + } + + for _, txin := range tx.MsgTx().TxIn { + if bf.containsOutPoint(&txin.PreviousOutpoint) { + return true + } + pushedData, err := btcscript.PushedData(txin.SignatureScript) + if err != nil { + break + } + for _, p := range pushedData { + if bf.contains(p) { + return true + } + } + } + return false +} + +// MatchesTx returns true if the BloomFilter matches data within the passed transaction, +// otherwise false is returned. If the BloomFilter does match the passed transaction, +// it will also update the BloomFilter if required. +func (bf *BloomFilter) MatchesTx(tx *btcutil.Tx) bool { + bf.Lock() + defer bf.Unlock() + + return bf.matches(tx) +} + +// MsgFilterLoad returns the underlying btcwire.MsgFilterLoad for the BloomFilter. +func (bf *BloomFilter) MsgFilterLoad() *btcwire.MsgFilterLoad { + return bf.msgFilterLoad +} + +func (bf *BloomFilter) hash(hashNum uint32, data []byte) uint32 { + // bitcoind: 0xFBA4C795 chosen as it guarantees a reasonable bit + // difference between nHashNum values. + mm := MurmurHash3(hashNum*0xFBA4C795+bf.msgFilterLoad.Tweak, data) + return mm % (uint32(len(bf.msgFilterLoad.Filter)) * 8) +} + +// min is a convenience function to return the minimum value of the two +// passed uint32 values. +func min(a, b uint32) uint32 { + if a < b { + return a + } + return b +} diff --git a/bloomfilter/filter_test.go b/bloomfilter/filter_test.go new file mode 100644 index 0000000..6fdaea2 --- /dev/null +++ b/bloomfilter/filter_test.go @@ -0,0 +1,554 @@ +package bloomfilter_test + +import ( + "bytes" + "encoding/hex" + "github.com/conformal/btcutil" + "github.com/conformal/btcutil/bloomfilter" + "github.com/conformal/btcwire" + "testing" +) + +func TestFilterLarge(t *testing.T) { + f := bloomfilter.New(100000000, 0, 0.01, btcwire.BloomUpdateNone) + + if len(f.MsgFilterLoad().Filter) > btcwire.MaxFilterLoadFilterSize { + t.Errorf("TestFilterLarge test failed: %d > %d", + len(f.MsgFilterLoad().Filter), btcwire.MaxFilterLoadFilterSize) + } +} + +func TestFilterInsert1(t *testing.T) { + var tests = []struct { + hex string + insert bool + }{ + {"99108ad8ed9bb6274d3980bab5a85c048f0950c8", true}, + {"19108ad8ed9bb6274d3980bab5a85c048f0950c8", false}, + {"b5a2c786d9ef4658287ced5914b37a1b4aa32eee", true}, + {"b9300670b4c5366e95b2699e8b18bc75e5f729c5", true}, + } + + f := bloomfilter.New(3, 0, 0.01, btcwire.BloomUpdateAll) + + for x, test := range tests { + data, err := hex.DecodeString(test.hex) + if err != nil { + t.Errorf("TestFilterInsert1 DecodeString failed: %v\n", err) + return + } + if test.insert { + f.Add(data) + } + + result := f.Contains(data) + if test.insert != result { + t.Errorf("TestFilterInsert1 Contains test #%d failure: got %v want %v\n", + x, result, test.insert) + return + } + } + + want, err := hex.DecodeString("03614e9b050000000000000001") + if err != nil { + t.Errorf("TestFilterInsert1 DecodeString failed: %v\n", err) + return + } + + got := bytes.NewBuffer(nil) + err = f.MsgFilterLoad().BtcEncode(got, btcwire.ProtocolVersion) + if err != nil { + t.Errorf("TestFilterInsert1 BtcDecode failed: %v\n", err) + return + } + + if !bytes.Equal(got.Bytes(), want) { + t.Errorf("TestFilterInsert1 failure: got %v want %v\n", + got.Bytes(), want) + return + } +} + +func TestFilterInsertWithTweak(t *testing.T) { + var tests = []struct { + hex string + insert bool + }{ + {"99108ad8ed9bb6274d3980bab5a85c048f0950c8", true}, + {"19108ad8ed9bb6274d3980bab5a85c048f0950c8", false}, + {"b5a2c786d9ef4658287ced5914b37a1b4aa32eee", true}, + {"b9300670b4c5366e95b2699e8b18bc75e5f729c5", true}, + } + + f := bloomfilter.New(3, 2147483649, 0.01, btcwire.BloomUpdateAll) + + for x, test := range tests { + data, err := hex.DecodeString(test.hex) + if err != nil { + t.Errorf("TestFilterInsertWithTweak DecodeString failed: %v\n", err) + return + } + if test.insert { + f.Add(data) + } + + result := f.Contains(data) + if test.insert != result { + t.Errorf("TestFilterInsertWithTweak Contains test #%d failure: got %v want %v\n", + x, result, test.insert) + return + } + } + + want, err := hex.DecodeString("03ce4299050000000100008001") + if err != nil { + t.Errorf("TestFilterInsertWithTweak DecodeString failed: %v\n", err) + return + } + got := bytes.NewBuffer(nil) + err = f.MsgFilterLoad().BtcEncode(got, btcwire.ProtocolVersion) + if err != nil { + t.Errorf("TestFilterInsertWithTweak BtcDecode failed: %v\n", err) + return + } + + if !bytes.Equal(got.Bytes(), want) { + t.Errorf("TestFilterInsertWithTweak failure: got %v want %v\n", + got.Bytes(), want) + return + } +} + +func TestFilterInsertKey(t *testing.T) { + secret := "5Kg1gnAjaLfKiwhhPpGS3QfRg2m6awQvaj98JCZBZQ5SuS2F15C" + + wif, err := btcutil.DecodeWIF(secret) + if err != nil { + t.Errorf("TestFilterInsertKey DecodeWIF failed: %v", err) + return + } + + + f := bloomfilter.New(2, 0, 0.001, btcwire.BloomUpdateAll) + f.Add(wif.SerializePubKey()) + f.Add(btcutil.Hash160(wif.SerializePubKey())) + + want, err := hex.DecodeString("038fc16b080000000000000001") + if err != nil { + t.Errorf("TestFilterInsertWithTweak DecodeString failed: %v\n", err) + return + } + got := bytes.NewBuffer(nil) + err = f.MsgFilterLoad().BtcEncode(got, btcwire.ProtocolVersion) + if err != nil { + t.Errorf("TestFilterInsertWithTweak BtcDecode failed: %v\n", err) + return + } + + if !bytes.Equal(got.Bytes(), want) { + t.Errorf("TestFilterInsertWithTweak failure: got %v want %v\n", + got.Bytes(), want) + return + } +} + +func TestFilterBloomMatch(t *testing.T) { + str := "01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e" + + "88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7" + + "c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d2" + + "7d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee" + + "51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5e" + + "eef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c33" + + "9ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3" + + "bc6e2754dbcff1988ac2f15de00000000001976a914a266436d296554760" + + "8b9e15d9032a7b9d64fa43188ac00000000" + strBytes, err := hex.DecodeString(str) + if err != nil { + t.Errorf("TestFilterBloomMatch DecodeString failure: %v", err) + return + } + tx, err := btcutil.NewTxFromBytes(strBytes) + if err != nil { + t.Errorf("TestFilterBloomMatch NewTxFromBytes failure: %v", err) + return + } + spendingTxBytes := []byte{0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, + 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, + 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, + 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, + 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, + 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, + 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, + 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, + 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, + 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, + 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, + 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, + 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, + 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, + 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, + 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, + 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, + 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, + 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, + 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, + 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, + 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, + 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, + 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, + 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, + 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, + 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, + 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00} + + spendingTx, err := btcutil.NewTxFromBytes(spendingTxBytes) + if err != nil { + t.Errorf("TestFilterBloomMatch NewTxFromBytes failure: %v", err) + return + } + + f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr := "b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b" + sha, err := btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch NewShaHashFromStr failed: %v\n", err) + return + } + f.AddShaHash(sha) + if !f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch didn't match sha %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "6bff7fcd4f8565ef406dd5d63d4ff94f318fe82027fd4dc451b04474019f74b4" + shaBytes, err := hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch DecodeString failed: %v\n", err) + return + } + f.Add(shaBytes) + if !f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch didn't match sha %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "30450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065" + + "f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643" + + "ac4cb7cb3c462aced7f14711a01" + shaBytes, err = hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch DecodeString failed: %v\n", err) + return + } + f.Add(shaBytes) + if !f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch didn't match input signature %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95" + + "c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe" + + "76036c339" + shaBytes, err = hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch DecodeString failed: %v\n", err) + return + } + f.Add(shaBytes) + if !f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch didn't match input pubkey %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "04943fdd508053c75000106d3bc6e2754dbcff19" + shaBytes, err = hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch DecodeString failed: %v\n", err) + return + } + f.Add(shaBytes) + if !f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch didn't match output address %s", inputStr) + } + if !f.MatchesTx(spendingTx) { + t.Errorf("TestFilterBloomMatch spendingTx didn't match output address %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "a266436d2965547608b9e15d9032a7b9d64fa431" + shaBytes, err = hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch DecodeString failed: %v\n", err) + return + } + f.Add(shaBytes) + if !f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch didn't match output address %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b" + sha, err = btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch NewShaHashFromStr failed: %v\n", err) + return + } + outpoint := btcwire.NewOutPoint(sha, 0) + f.AddOutPoint(outpoint) + if !f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch didn't match outpoint %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "00000009e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436" + sha, err = btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch NewShaHashFromStr failed: %v\n", err) + return + } + f.AddShaHash(sha) + if f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch matched sha %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "0000006d2965547608b9e15d9032a7b9d64fa431" + shaBytes, err = hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch DecodeString failed: %v\n", err) + return + } + f.Add(shaBytes) + if f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch matched address %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b" + sha, err = btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch NewShaHashFromStr failed: %v\n", err) + return + } + outpoint = btcwire.NewOutPoint(sha, 1) + f.AddOutPoint(outpoint) + if f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch matched outpoint %s", inputStr) + } + + f = bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateAll) + inputStr = "000000d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b" + sha, err = btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestFilterBloomMatch NewShaHashFromStr failed: %v\n", err) + return + } + outpoint = btcwire.NewOutPoint(sha, 0) + f.AddOutPoint(outpoint) + if f.MatchesTx(tx) { + t.Errorf("TestFilterBloomMatch matched outpoint %s", inputStr) + } +} + +func TestFilterInsertUpdateNone(t *testing.T) { + f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateNone) + + // Add the generation pubkey + inputStr := "04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c" + + "876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a" + + "2252247d97a46a91" + inputBytes, err := hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterInsertUpdateNone DecodeString failed: %v", err) + return + } + f.Add(inputBytes) + + // Add the output address for the 4th transaction + inputStr = "b6efd80d99179f4f4ff6f4dd0a007d018c385d21" + inputBytes, err = hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterInsertUpdateNone DecodeString failed: %v", err) + return + } + f.Add(inputBytes) + + inputStr = "147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b" + sha, err := btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestFilterInsertUpdateNone NewShaHashFromStr failed: %v", err) + return + } + outpoint := btcwire.NewOutPoint(sha, 0) + + if f.ContainsOutPoint(outpoint) { + t.Errorf("TestFilterInsertUpdateNone matched outpoint %s", inputStr) + return + } + + inputStr = "02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041" + sha, err = btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestFilterInsertUpdateNone NewShaHashFromStr failed: %v", err) + return + } + outpoint = btcwire.NewOutPoint(sha, 0) + + if f.ContainsOutPoint(outpoint) { + t.Errorf("TestFilterInsertUpdateNone matched outpoint %s", inputStr) + return + } +} + +func TestFilterInsertP2PubKeyOnly(t *testing.T) { + blockStr := "0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc" + + "880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367" + + "117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010" + + "00000000000000000000000000000000000000000000000000000000000" + + "0000ffffffff07044c86041b0136ffffffff0100f2052a0100000043410" + + "4eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2" + + "c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a22522" + + "47d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255" + + "120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356" + + "e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062e" + + "a10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa" + + "608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ec" + + "bba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141" + + "b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac0000000001000000" + + "03fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26a" + + "f16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23" + + "b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6" + + "b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e" + + "29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245b" + + "d69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585" + + "caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e7" + + "5429df397b5af83000000004948304502202bdb79c596a9ffc24e96f438" + + "6199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b72" + + "4fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffff" + + "ff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cb" + + "cb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9cecc" + + "d328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f11" + + "87779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff010071" + + "4460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8" + + "d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397" + + "cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3" + + "e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e4022100" + + "8581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7" + + "c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf64852" + + "61c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d" + + "3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f2" + + "48e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124" + + "c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e93" + + "0220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346" + + "669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6" + + "485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270e" + + "fb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051" + + "fd372bb7a537232946e0a46f53636b4dafdaa4000000008c49304602210" + + "0c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de" + + "35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46f" + + "c37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0" + + "f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f" + + "4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f8" + + "94aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493" + + "046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8a" + + "a788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415" + + "588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb" + + "7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018" + + "ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8" + + "040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188a" + + "c000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758d" + + "f616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34" + + "fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243" + + "bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec" + + "20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442" + + "e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632" + + "d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10" + + "eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a91" + + "4a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000" + + "000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850" + + "927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec" + + "2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a25" + + "7b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e5" + + "21fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455f" + + "e30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098" + + "544bffffffff0240420f00000000001976a9144676d1b820d63ec272f19" + + "00d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00" + + "d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc91750" + + "1ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f10000" + + "00008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e" + + "280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba80" + + "7892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b" + + "5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c4" + + "7d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41e" + + "d70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d0" + + "68000000008b4830450221008513ad65187b903aed1102d1d0c47688127" + + "658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de6603" + + "5fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50b" + + "e1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b0682" + + "0edca9ef982c35fda2d255afba340068c5035552368bc7200c1488fffff" + + "fff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e40" + + "0f8699eb4888ac00000000" + blockBytes, err := hex.DecodeString(blockStr) + if err != nil { + t.Errorf("TestFilterInsertP2PubKeyOnly DecodeString failed: %v", err) + return + } + block, err := btcutil.NewBlockFromBytes(blockBytes) + if err != nil { + t.Errorf("TestFilterInsertP2PubKeyOnly NewBlockFromBytes failed: %v", err) + return + } + + f := bloomfilter.New(10, 0, 0.000001, btcwire.BloomUpdateP2PubkeyOnly) + + // Generation pubkey + inputStr := "04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c" + + "876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a" + + "2252247d97a46a91" + inputBytes, err := hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterInsertP2PubKeyOnly DecodeString failed: %v", err) + return + } + f.Add(inputBytes) + + // Output address of 4th transaction + inputStr = "b6efd80d99179f4f4ff6f4dd0a007d018c385d21" + inputBytes, err = hex.DecodeString(inputStr) + if err != nil { + t.Errorf("TestFilterInsertP2PubKeyOnly DecodeString failed: %v", err) + return + } + f.Add(inputBytes) + + // Ignore return value -- this is just used to update the filter. + _, _ = bloomfilter.NewMerkleBlock(block, f) + + // We should match the generation pubkey + inputStr = "147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b" + sha, err := btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestMerkleBlockP2PubKeyOnly NewShaHashFromStr failed: %v", err) + return + } + outpoint := btcwire.NewOutPoint(sha, 0) + if !f.ContainsOutPoint(outpoint) { + t.Errorf("TestMerkleBlockP2PubKeyOnly didn't match the generation "+ + "outpoint %s", inputStr) + return + } + + // We should not match the 4th transaction, which is not p2pk + inputStr = "02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041" + sha, err = btcwire.NewShaHashFromStr(inputStr) + if err != nil { + t.Errorf("TestMerkleBlockP2PubKeyOnly NewShaHashFromStr failed: %v", err) + return + } + outpoint = btcwire.NewOutPoint(sha, 0) + if f.ContainsOutPoint(outpoint) { + t.Errorf("TestMerkleBlockP2PubKeyOnly matched outpoint %s", inputStr) + return + } +} diff --git a/bloomfilter/merkleblock.go b/bloomfilter/merkleblock.go new file mode 100644 index 0000000..11d8c82 --- /dev/null +++ b/bloomfilter/merkleblock.go @@ -0,0 +1,89 @@ +package bloomfilter + +import ( + "github.com/conformal/btcchain" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" +) + +type merkleBlock struct { + msgMerkleBlock *btcwire.MsgMerkleBlock + allHashes []*btcwire.ShaHash + finalHashes []*btcwire.ShaHash + matchedBits []byte + bits []byte +} + +// NewMerkleBlock returns a new *btcwire.MsgMerkleBlock based on the passed +// block and filter. +func NewMerkleBlock(block *btcutil.Block, filter *BloomFilter) (*btcwire.MsgMerkleBlock, []*btcwire.ShaHash) { + blockHeader := block.MsgBlock().Header + mBlock := merkleBlock{ + msgMerkleBlock: btcwire.NewMsgMerkleBlock(&blockHeader), + } + + var matchedHashes []*btcwire.ShaHash + for _, tx := range block.Transactions() { + if filter.MatchesTx(tx) { + mBlock.matchedBits = append(mBlock.matchedBits, 0x01) + matchedHashes = append(matchedHashes, tx.Sha()) + } else { + mBlock.matchedBits = append(mBlock.matchedBits, 0x00) + } + mBlock.allHashes = append(mBlock.allHashes, tx.Sha()) + } + mBlock.msgMerkleBlock.Transactions = uint32(len(block.Transactions())) + + height := uint32(0) + for mBlock.calcTreeWidth(height) > 1 { + height++ + } + + mBlock.traverseAndBuild(height, 0) + + for _, sha := range mBlock.finalHashes { + mBlock.msgMerkleBlock.AddTxHash(sha) + } + mBlock.msgMerkleBlock.Flags = make([]byte, (len(mBlock.bits)+7)/8) + for i := uint32(0); i < uint32(len(mBlock.bits)); i++ { + mBlock.msgMerkleBlock.Flags[i/8] |= mBlock.bits[i] << (i % 8) + } + return mBlock.msgMerkleBlock, matchedHashes +} + +func (m *merkleBlock) calcTreeWidth(height uint32) uint32 { + return (m.msgMerkleBlock.Transactions + (1 << height) - 1) >> height +} + +func (m *merkleBlock) calcHash(height, pos uint32) *btcwire.ShaHash { + if height == 0 { + return m.allHashes[pos] + } else { + var right *btcwire.ShaHash + left := m.calcHash(height-1, pos*2) + if pos*2+1 < m.calcTreeWidth(height-1) { + right = m.calcHash(height-1, pos*2+1) + } else { + right = left + } + return btcchain.HashMerkleBranches(left, right) + } +} + +func (m *merkleBlock) traverseAndBuild(height, pos uint32) { + var isParent byte + + for i := pos << height; i < (pos+1)<> (32 - 15)) + k1 *= c2 + h1 ^= k1 + h1 = (h1 << 13) | (h1 >> (32 - 13)) + h1 = h1*5 + 0xe6546b64 + } + + // tail + tailidx := numBlocks * 4 + k1 = 0 + + switch dataLen & 3 { + case 3: + k1 ^= uint32(data[tailidx+2]) << 16 + fallthrough + case 2: + k1 ^= uint32(data[tailidx+1]) << 8 + fallthrough + case 1: + k1 ^= uint32(data[tailidx]) + k1 *= c1 + k1 = (k1 << 15) | (k1 >> (32 - 15)) + k1 *= c2 + h1 ^= k1 + } + + // Finalization + h1 ^= uint32(dataLen) + h1 ^= h1 >> 16 + h1 *= 0x85ebca6b + h1 ^= h1 >> 13 + h1 *= 0xc2b2ae35 + h1 ^= h1 >> 16 + + return h1 +} diff --git a/bloomfilter/murmurhash3_test.go b/bloomfilter/murmurhash3_test.go new file mode 100644 index 0000000..861951e --- /dev/null +++ b/bloomfilter/murmurhash3_test.go @@ -0,0 +1,38 @@ +package bloomfilter_test + +import ( + "github.com/conformal/btcutil/bloomfilter" + "testing" +) + +func TestMurmurHash3(t *testing.T) { + var tests = []struct { + seed uint32 + data []byte + out uint32 + }{ + {0x00000000, []byte{}, 0x00000000}, + {0xfba4c795, []byte{}, 0x6a396f08}, + {0xffffffff, []byte{}, 0x81f16f39}, + {0x00000000, []byte{0x00}, 0x514e28b7}, + {0xfba4c795, []byte{0x00}, 0xea3f0b17}, + {0x00000000, []byte{0xff}, 0xfd6cf10d}, + {0x00000000, []byte{0x00, 0x11}, 0x16c6b7ab}, + {0x00000000, []byte{0x00, 0x11, 0x22}, 0x8eb51c3d}, + {0x00000000, []byte{0x00, 0x11, 0x22, 0x33}, 0xb4471bf8}, + {0x00000000, []byte{0x00, 0x11, 0x22, 0x33, 0x44}, 0xe2301fa8}, + {0x00000000, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}, 0xfc2e4a15}, + {0x00000000, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66}, 0xb074502c}, + {0x00000000, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77}, 0x8034d2a0}, + {0x00000000, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88}, 0xb4698def}, + } + + for x, test := range tests { + result := bloomfilter.MurmurHash3(test.seed, test.data) + if result != test.out { + t.Errorf("MurmurHash3 test #%d failed: got %v want %v\n", + x, result, test.out) + continue + } + } +} diff --git a/bloomfilter/test_coverage.txt b/bloomfilter/test_coverage.txt new file mode 100644 index 0000000..9bd4eb4 --- /dev/null +++ b/bloomfilter/test_coverage.txt @@ -0,0 +1,26 @@ + +github.com/conformal/btcutil/bloomfilter/murmurhash3.go MurmurHash3 100.00% (34/34) +github.com/conformal/btcutil/bloomfilter/merkleblock.go NewMerkleBlock 100.00% (20/20) +github.com/conformal/btcutil/bloomfilter/merkleblock.go merkleBlock.traverseAndBuild 100.00% (9/9) +github.com/conformal/btcutil/bloomfilter/merkleblock.go merkleBlock.calcHash 100.00% (8/8) +github.com/conformal/btcutil/bloomfilter/filter.go New 100.00% (6/6) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.containsOutPoint 100.00% (4/4) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.addOutPoint 100.00% (4/4) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.Add 100.00% (3/3) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.AddShaHash 100.00% (3/3) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.Contains 100.00% (3/3) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.ContainsOutPoint 100.00% (3/3) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.MatchesTx 100.00% (3/3) +github.com/conformal/btcutil/bloomfilter/filter.go min 100.00% (3/3) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.AddOutPoint 100.00% (3/3) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.hash 100.00% (2/2) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.MsgFilterLoad 100.00% (1/1) +github.com/conformal/btcutil/bloomfilter/merkleblock.go merkleBlock.calcTreeWidth 100.00% (1/1) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.matches 92.86% (26/28) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.contains 85.71% (6/7) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.add 80.00% (4/5) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.Unload 0.00% (0/3) +github.com/conformal/btcutil/bloomfilter/filter.go BloomFilter.IsLoaded 0.00% (0/3) +github.com/conformal/btcutil/bloomfilter/filter.go Load 0.00% (0/1) +github.com/conformal/btcutil/bloomfilter ---------------------------- 92.99% (146/157) +