2016-08-08 19:38:16 +02:00
|
|
|
// Copyright (c) 2014-2016 The btcsuite developers
|
2014-07-09 04:59:33 +02:00
|
|
|
// Use of this source code is governed by an ISC
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
package bloom
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/binary"
|
|
|
|
"math"
|
|
|
|
"sync"
|
|
|
|
|
2021-09-10 22:30:39 +02:00
|
|
|
"github.com/lbryio/lbcd/chaincfg/chainhash"
|
|
|
|
"github.com/lbryio/lbcd/txscript"
|
|
|
|
"github.com/lbryio/lbcd/wire"
|
|
|
|
"github.com/lbryio/lbcutil"
|
2014-07-09 04:59:33 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// ln2Squared is simply the square of the natural log of 2.
|
|
|
|
const ln2Squared = math.Ln2 * math.Ln2
|
|
|
|
|
|
|
|
// minUint32 is a convenience function to return the minimum value of the two
|
|
|
|
// passed uint32 values.
|
|
|
|
func minUint32(a, b uint32) uint32 {
|
|
|
|
if a < b {
|
|
|
|
return a
|
|
|
|
}
|
|
|
|
return b
|
|
|
|
}
|
|
|
|
|
|
|
|
// Filter defines a bitcoin bloom filter that provides easy manipulation of raw
|
|
|
|
// filter data.
|
|
|
|
type Filter struct {
|
2014-09-01 04:49:23 +02:00
|
|
|
mtx sync.Mutex
|
2015-02-05 21:48:38 +01:00
|
|
|
msgFilterLoad *wire.MsgFilterLoad
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewFilter creates a new bloom filter instance, mainly to be used by SPV
|
|
|
|
// clients. The tweak parameter is a random value added to the seed value.
|
|
|
|
// The false positive rate is the probability of a false positive where 1.0 is
|
|
|
|
// "match everything" and zero is unachievable. Thus, providing any false
|
|
|
|
// positive rates less than 0 or greater than 1 will be adjusted to the valid
|
|
|
|
// range.
|
|
|
|
//
|
|
|
|
// For more information on what values to use for both elements and fprate,
|
|
|
|
// see https://en.wikipedia.org/wiki/Bloom_filter.
|
2015-02-05 21:48:38 +01:00
|
|
|
func NewFilter(elements, tweak uint32, fprate float64, flags wire.BloomUpdateType) *Filter {
|
2014-07-09 04:59:33 +02:00
|
|
|
// Massage the false positive rate to sane values.
|
|
|
|
if fprate > 1.0 {
|
|
|
|
fprate = 1.0
|
|
|
|
}
|
2016-02-26 04:09:38 +01:00
|
|
|
if fprate < 1e-9 {
|
2014-07-09 04:59:33 +02:00
|
|
|
fprate = 1e-9
|
|
|
|
}
|
|
|
|
|
|
|
|
// Calculate the size of the filter in bytes for the given number of
|
|
|
|
// elements and false positive rate.
|
|
|
|
//
|
|
|
|
// Equivalent to m = -(n*ln(p) / ln(2)^2), where m is in bits.
|
|
|
|
// Then clamp it to the maximum filter size and convert to bytes.
|
|
|
|
dataLen := uint32(-1 * float64(elements) * math.Log(fprate) / ln2Squared)
|
2015-02-05 21:48:38 +01:00
|
|
|
dataLen = minUint32(dataLen, wire.MaxFilterLoadFilterSize*8) / 8
|
2014-07-09 04:59:33 +02:00
|
|
|
|
|
|
|
// Calculate the number of hash functions based on the size of the
|
|
|
|
// filter calculated above and the number of elements.
|
|
|
|
//
|
|
|
|
// Equivalent to k = (m/n) * ln(2)
|
|
|
|
// Then clamp it to the maximum allowed hash funcs.
|
|
|
|
hashFuncs := uint32(float64(dataLen*8) / float64(elements) * math.Ln2)
|
2015-02-05 21:48:38 +01:00
|
|
|
hashFuncs = minUint32(hashFuncs, wire.MaxFilterLoadHashFuncs)
|
2014-07-09 04:59:33 +02:00
|
|
|
|
|
|
|
data := make([]byte, dataLen)
|
2015-02-05 21:48:38 +01:00
|
|
|
msg := wire.NewMsgFilterLoad(data, hashFuncs, tweak, flags)
|
2014-07-09 04:59:33 +02:00
|
|
|
|
|
|
|
return &Filter{
|
|
|
|
msgFilterLoad: msg,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// LoadFilter creates a new Filter instance with the given underlying
|
2015-02-05 21:48:38 +01:00
|
|
|
// wire.MsgFilterLoad.
|
|
|
|
func LoadFilter(filter *wire.MsgFilterLoad) *Filter {
|
2014-07-09 04:59:33 +02:00
|
|
|
return &Filter{
|
|
|
|
msgFilterLoad: filter,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// IsLoaded returns true if a filter is loaded, otherwise false.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
|
|
|
func (bf *Filter) IsLoaded() bool {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
|
|
|
loaded := bf.msgFilterLoad != nil
|
|
|
|
bf.mtx.Unlock()
|
|
|
|
return loaded
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
2014-08-30 04:02:52 +02:00
|
|
|
// Reload loads a new filter replacing any existing filter.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
2015-02-05 21:48:38 +01:00
|
|
|
func (bf *Filter) Reload(filter *wire.MsgFilterLoad) {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
2014-08-30 04:02:52 +02:00
|
|
|
bf.msgFilterLoad = filter
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Unlock()
|
2014-08-30 04:02:52 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Unload unloads the bloom filter.
|
2014-07-09 04:59:33 +02:00
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
|
|
|
func (bf *Filter) Unload() {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
2014-07-09 04:59:33 +02:00
|
|
|
bf.msgFilterLoad = nil
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Unlock()
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// hash returns the bit offset in the bloom filter which corresponds to the
|
2019-07-02 10:46:07 +02:00
|
|
|
// passed data for the given independent hash function number.
|
2014-07-09 04:59:33 +02:00
|
|
|
func (bf *Filter) hash(hashNum uint32, data []byte) uint32 {
|
|
|
|
// bitcoind: 0xfba4c795 chosen as it guarantees a reasonable bit
|
|
|
|
// difference between hashNum values.
|
|
|
|
//
|
|
|
|
// Note that << 3 is equivalent to multiplying by 8, but is faster.
|
|
|
|
// Thus the returned hash is brought into range of the number of bits
|
|
|
|
// the filter has and returned.
|
|
|
|
mm := MurmurHash3(hashNum*0xfba4c795+bf.msgFilterLoad.Tweak, data)
|
|
|
|
return mm % (uint32(len(bf.msgFilterLoad.Filter)) << 3)
|
|
|
|
}
|
|
|
|
|
|
|
|
// matches returns true if the bloom filter might contain the passed data and
|
|
|
|
// false if it definitely does not.
|
|
|
|
//
|
|
|
|
// This function MUST be called with the filter lock held.
|
|
|
|
func (bf *Filter) matches(data []byte) bool {
|
|
|
|
if bf.msgFilterLoad == nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// The bloom filter does not contain the data if any of the bit offsets
|
|
|
|
// which result from hashing the data using each independent hash
|
|
|
|
// function are not set. The shifts and masks below are a faster
|
|
|
|
// equivalent of:
|
|
|
|
// arrayIndex := idx / 8 (idx >> 3)
|
|
|
|
// bitOffset := idx % 8 (idx & 7)
|
|
|
|
/// if filter[arrayIndex] & 1<<bitOffset == 0 { ... }
|
|
|
|
for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
|
|
|
|
idx := bf.hash(i, data)
|
|
|
|
if bf.msgFilterLoad.Filter[idx>>3]&(1<<(idx&7)) == 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// Matches returns true if the bloom filter might contain the passed data and
|
|
|
|
// false if it definitely does not.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
|
|
|
func (bf *Filter) Matches(data []byte) bool {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
|
|
|
match := bf.matches(data)
|
|
|
|
bf.mtx.Unlock()
|
|
|
|
return match
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// matchesOutPoint returns true if the bloom filter might contain the passed
|
|
|
|
// outpoint and false if it definitely does not.
|
|
|
|
//
|
|
|
|
// This function MUST be called with the filter lock held.
|
2015-02-05 21:48:38 +01:00
|
|
|
func (bf *Filter) matchesOutPoint(outpoint *wire.OutPoint) bool {
|
2014-07-09 04:59:33 +02:00
|
|
|
// Serialize
|
2016-08-08 19:38:16 +02:00
|
|
|
var buf [chainhash.HashSize + 4]byte
|
2016-04-23 10:34:57 +02:00
|
|
|
copy(buf[:], outpoint.Hash[:])
|
2016-08-08 19:38:16 +02:00
|
|
|
binary.LittleEndian.PutUint32(buf[chainhash.HashSize:], outpoint.Index)
|
2014-07-09 04:59:33 +02:00
|
|
|
|
|
|
|
return bf.matches(buf[:])
|
|
|
|
}
|
|
|
|
|
|
|
|
// MatchesOutPoint returns true if the bloom filter might contain the passed
|
|
|
|
// outpoint and false if it definitely does not.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
2015-02-05 21:48:38 +01:00
|
|
|
func (bf *Filter) MatchesOutPoint(outpoint *wire.OutPoint) bool {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
|
|
|
match := bf.matchesOutPoint(outpoint)
|
|
|
|
bf.mtx.Unlock()
|
|
|
|
return match
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// add adds the passed byte slice to the bloom filter.
|
|
|
|
//
|
|
|
|
// This function MUST be called with the filter lock held.
|
|
|
|
func (bf *Filter) add(data []byte) {
|
|
|
|
if bf.msgFilterLoad == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Adding data to a bloom filter consists of setting all of the bit
|
|
|
|
// offsets which result from hashing the data using each independent
|
|
|
|
// hash function. The shifts and masks below are a faster equivalent
|
|
|
|
// of:
|
|
|
|
// arrayIndex := idx / 8 (idx >> 3)
|
|
|
|
// bitOffset := idx % 8 (idx & 7)
|
|
|
|
/// filter[arrayIndex] |= 1<<bitOffset
|
|
|
|
for i := uint32(0); i < bf.msgFilterLoad.HashFuncs; i++ {
|
|
|
|
idx := bf.hash(i, data)
|
|
|
|
bf.msgFilterLoad.Filter[idx>>3] |= (1 << (7 & idx))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add adds the passed byte slice to the bloom filter.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
|
|
|
func (bf *Filter) Add(data []byte) {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
2014-07-09 04:59:33 +02:00
|
|
|
bf.add(data)
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Unlock()
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
2016-08-08 19:38:16 +02:00
|
|
|
// AddHash adds the passed chainhash.Hash to the Filter.
|
2014-07-09 04:59:33 +02:00
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
2016-08-08 19:38:16 +02:00
|
|
|
func (bf *Filter) AddHash(hash *chainhash.Hash) {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
2016-04-23 10:34:57 +02:00
|
|
|
bf.add(hash[:])
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Unlock()
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// addOutPoint adds the passed transaction outpoint to the bloom filter.
|
|
|
|
//
|
|
|
|
// This function MUST be called with the filter lock held.
|
2015-02-05 21:48:38 +01:00
|
|
|
func (bf *Filter) addOutPoint(outpoint *wire.OutPoint) {
|
2014-07-09 04:59:33 +02:00
|
|
|
// Serialize
|
2016-08-08 19:38:16 +02:00
|
|
|
var buf [chainhash.HashSize + 4]byte
|
2016-04-23 10:34:57 +02:00
|
|
|
copy(buf[:], outpoint.Hash[:])
|
2016-08-08 19:38:16 +02:00
|
|
|
binary.LittleEndian.PutUint32(buf[chainhash.HashSize:], outpoint.Index)
|
2014-07-09 04:59:33 +02:00
|
|
|
|
|
|
|
bf.add(buf[:])
|
|
|
|
}
|
|
|
|
|
|
|
|
// AddOutPoint adds the passed transaction outpoint to the bloom filter.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
2015-02-05 21:48:38 +01:00
|
|
|
func (bf *Filter) AddOutPoint(outpoint *wire.OutPoint) {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
2014-07-09 04:59:33 +02:00
|
|
|
bf.addOutPoint(outpoint)
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Unlock()
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// maybeAddOutpoint potentially adds the passed outpoint to the bloom filter
|
|
|
|
// depending on the bloom update flags and the type of the passed public key
|
|
|
|
// script.
|
|
|
|
//
|
|
|
|
// This function MUST be called with the filter lock held.
|
2016-08-08 19:38:16 +02:00
|
|
|
func (bf *Filter) maybeAddOutpoint(pkScript []byte, outHash *chainhash.Hash, outIdx uint32) {
|
2014-07-09 04:59:33 +02:00
|
|
|
switch bf.msgFilterLoad.Flags {
|
2015-02-05 21:48:38 +01:00
|
|
|
case wire.BloomUpdateAll:
|
|
|
|
outpoint := wire.NewOutPoint(outHash, outIdx)
|
2014-07-09 04:59:33 +02:00
|
|
|
bf.addOutPoint(outpoint)
|
2015-02-05 21:48:38 +01:00
|
|
|
case wire.BloomUpdateP2PubkeyOnly:
|
2015-01-30 19:11:32 +01:00
|
|
|
class := txscript.GetScriptClass(pkScript)
|
|
|
|
if class == txscript.PubKeyTy || class == txscript.MultiSigTy {
|
2015-02-05 21:48:38 +01:00
|
|
|
outpoint := wire.NewOutPoint(outHash, outIdx)
|
2014-07-09 04:59:33 +02:00
|
|
|
bf.addOutPoint(outpoint)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// matchTxAndUpdate returns true if the bloom filter matches data within the
|
|
|
|
// passed transaction, otherwise false is returned. If the filter does match
|
|
|
|
// the passed transaction, it will also update the filter depending on the bloom
|
|
|
|
// update flags set via the loaded filter if needed.
|
|
|
|
//
|
|
|
|
// This function MUST be called with the filter lock held.
|
2021-09-10 22:30:39 +02:00
|
|
|
func (bf *Filter) matchTxAndUpdate(tx *lbcutil.Tx) bool {
|
2014-07-09 04:59:33 +02:00
|
|
|
// Check if the filter matches the hash of the transaction.
|
|
|
|
// This is useful for finding transactions when they appear in a block.
|
2016-08-08 19:38:16 +02:00
|
|
|
matched := bf.matches(tx.Hash()[:])
|
2014-07-09 04:59:33 +02:00
|
|
|
|
|
|
|
// Check if the filter matches any data elements in the public key
|
|
|
|
// scripts of any of the outputs. When it does, add the outpoint that
|
|
|
|
// matched so transactions which spend from the matched transaction are
|
|
|
|
// also included in the filter. This removes the burden of updating the
|
|
|
|
// filter for this scenario from the client. It is also more efficient
|
|
|
|
// on the network since it avoids the need for another filteradd message
|
|
|
|
// from the client and avoids some potential races that could otherwise
|
|
|
|
// occur.
|
|
|
|
for i, txOut := range tx.MsgTx().TxOut {
|
2015-01-30 19:11:32 +01:00
|
|
|
pushedData, err := txscript.PushedData(txOut.PkScript)
|
2014-07-09 04:59:33 +02:00
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, data := range pushedData {
|
|
|
|
if !bf.matches(data) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
matched = true
|
2016-08-08 19:38:16 +02:00
|
|
|
bf.maybeAddOutpoint(txOut.PkScript, tx.Hash(), uint32(i))
|
2014-07-09 04:59:33 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Nothing more to do if a match has already been made.
|
|
|
|
if matched {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// At this point, the transaction and none of the data elements in the
|
|
|
|
// public key scripts of its outputs matched.
|
|
|
|
|
|
|
|
// Check if the filter matches any outpoints this transaction spends or
|
2019-07-02 10:46:07 +02:00
|
|
|
// any data elements in the signature scripts of any of the inputs.
|
2014-07-09 04:59:33 +02:00
|
|
|
for _, txin := range tx.MsgTx().TxIn {
|
2014-10-01 14:54:42 +02:00
|
|
|
if bf.matchesOutPoint(&txin.PreviousOutPoint) {
|
2014-07-09 04:59:33 +02:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2015-01-30 19:11:32 +01:00
|
|
|
pushedData, err := txscript.PushedData(txin.SignatureScript)
|
2014-07-09 04:59:33 +02:00
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, data := range pushedData {
|
|
|
|
if bf.matches(data) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// MatchTxAndUpdate returns true if the bloom filter matches data within the
|
|
|
|
// passed transaction, otherwise false is returned. If the filter does match
|
|
|
|
// the passed transaction, it will also update the filter depending on the bloom
|
|
|
|
// update flags set via the loaded filter if needed.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
2021-09-10 22:30:39 +02:00
|
|
|
func (bf *Filter) MatchTxAndUpdate(tx *lbcutil.Tx) bool {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
|
|
|
match := bf.matchTxAndUpdate(tx)
|
|
|
|
bf.mtx.Unlock()
|
|
|
|
return match
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|
|
|
|
|
2015-02-05 21:48:38 +01:00
|
|
|
// MsgFilterLoad returns the underlying wire.MsgFilterLoad for the bloom
|
2014-07-09 04:59:33 +02:00
|
|
|
// filter.
|
|
|
|
//
|
|
|
|
// This function is safe for concurrent access.
|
2015-02-05 21:48:38 +01:00
|
|
|
func (bf *Filter) MsgFilterLoad() *wire.MsgFilterLoad {
|
2014-09-01 04:49:23 +02:00
|
|
|
bf.mtx.Lock()
|
|
|
|
msg := bf.msgFilterLoad
|
|
|
|
bf.mtx.Unlock()
|
|
|
|
return msg
|
2014-07-09 04:59:33 +02:00
|
|
|
}
|