gcs: add some line spacing, wrap comments to 80 characters

This commit is contained in:
Olaoluwa Osuntokun 2017-04-27 20:34:50 -07:00
parent e3c79234e6
commit 0f2eb80fdb
5 changed files with 96 additions and 60 deletions

View file

@ -55,8 +55,8 @@ func DeriveKey(keyHash *chainhash.Hash) [gcs.KeySize]byte {
} }
// OutPointToFilterEntry is a utility function that derives a filter entry from // OutPointToFilterEntry is a utility function that derives a filter entry from
// a wire.OutPoint in a standardized way for use with both building and querying // a wire.OutPoint in a standardized way for use with both building and
// filters. // querying filters.
func OutPointToFilterEntry(outpoint wire.OutPoint) []byte { func OutPointToFilterEntry(outpoint wire.OutPoint) []byte {
// Size of the hash plus size of int32 index // Size of the hash plus size of int32 index
data := make([]byte, chainhash.HashSize+4) data := make([]byte, chainhash.HashSize+4)
@ -118,7 +118,7 @@ func (b *GCSBuilder) SetP(p uint8) *GCSBuilder {
// Preallocate sets the estimated filter size after calling Builder() to reduce // Preallocate sets the estimated filter size after calling Builder() to reduce
// the probability of memory reallocations. If the builder has already had data // the probability of memory reallocations. If the builder has already had data
// added to it, SetN has no effect. // added to it, Preallocate has no effect.
func (b *GCSBuilder) Preallocate(n uint32) *GCSBuilder { func (b *GCSBuilder) Preallocate(n uint32) *GCSBuilder {
// Do nothing if the builder's already errored out. // Do nothing if the builder's already errored out.
if b.err != nil { if b.err != nil {
@ -128,6 +128,7 @@ func (b *GCSBuilder) Preallocate(n uint32) *GCSBuilder {
if len(b.data) == 0 { if len(b.data) == 0 {
b.data = make([][]byte, 0, n) b.data = make([][]byte, 0, n)
} }
return b return b
} }
@ -157,8 +158,8 @@ func (b *GCSBuilder) AddEntries(data [][]byte) *GCSBuilder {
return b return b
} }
// AddOutPoint adds a wire.OutPoint to the list of entries to be included in the // AddOutPoint adds a wire.OutPoint to the list of entries to be included in
// GCS filter when it's built. // the GCS filter when it's built.
func (b *GCSBuilder) AddOutPoint(outpoint wire.OutPoint) *GCSBuilder { func (b *GCSBuilder) AddOutPoint(outpoint wire.OutPoint) *GCSBuilder {
// Do nothing if the builder's already errored out. // Do nothing if the builder's already errored out.
if b.err != nil { if b.err != nil {
@ -181,7 +182,7 @@ func (b *GCSBuilder) AddHash(hash *chainhash.Hash) *GCSBuilder {
// AddScript adds all the data pushed in the script serialized as the passed // AddScript adds all the data pushed in the script serialized as the passed
// []byte to the list of entries to be included in the GCS filter when it's // []byte to the list of entries to be included in the GCS filter when it's
// built. T // built.
func (b *GCSBuilder) AddScript(script []byte) *GCSBuilder { func (b *GCSBuilder) AddScript(script []byte) *GCSBuilder {
// Do nothing if the builder's already errored out. // Do nothing if the builder's already errored out.
if b.err != nil { if b.err != nil {
@ -204,16 +205,16 @@ func (b *GCSBuilder) Build() (*gcs.Filter, error) {
return gcs.BuildGCSFilter(b.p, b.key, b.data) return gcs.BuildGCSFilter(b.p, b.key, b.data)
} }
// WithKeyPN creates a GCSBuilder with specified key and the passed // WithKeyPN creates a GCSBuilder with specified key and the passed probability
// probability and estimated filter size. // and estimated filter size.
func WithKeyPN(key [gcs.KeySize]byte, p uint8, n uint32) *GCSBuilder { func WithKeyPN(key [gcs.KeySize]byte, p uint8, n uint32) *GCSBuilder {
b := GCSBuilder{} b := GCSBuilder{}
return b.SetKey(key).SetP(p).Preallocate(n) return b.SetKey(key).SetP(p).Preallocate(n)
} }
// WithKeyP creates a GCSBuilder with specified key and the passed // WithKeyP creates a GCSBuilder with specified key and the passed probability.
// probability. Estimated filter size is set to zero, which means more // Estimated filter size is set to zero, which means more reallocations are
// reallocations are done when building the filter. // done when building the filter.
func WithKeyP(key [gcs.KeySize]byte, p uint8) *GCSBuilder { func WithKeyP(key [gcs.KeySize]byte, p uint8) *GCSBuilder {
return WithKeyPN(key, p, 0) return WithKeyPN(key, p, 0)
} }
@ -246,8 +247,8 @@ func WithKeyHash(keyHash *chainhash.Hash) *GCSBuilder {
return WithKeyHashPN(keyHash, DefaultP, 0) return WithKeyHashPN(keyHash, DefaultP, 0)
} }
// WithRandomKeyPN creates a GCSBuilder with a cryptographically random // WithRandomKeyPN creates a GCSBuilder with a cryptographically random key and
// key and the passed probability and estimated filter size. // the passed probability and estimated filter size.
func WithRandomKeyPN(p uint8, n uint32) *GCSBuilder { func WithRandomKeyPN(p uint8, n uint32) *GCSBuilder {
key, err := RandomKey() key, err := RandomKey()
if err != nil { if err != nil {
@ -257,44 +258,63 @@ func WithRandomKeyPN(p uint8, n uint32) *GCSBuilder {
return WithKeyPN(key, p, n) return WithKeyPN(key, p, n)
} }
// WithRandomKeyP creates a GCSBuilder with a cryptographically random // WithRandomKeyP creates a GCSBuilder with a cryptographically random key and
// key and the passed probability. Estimated filter size is set to zero, which // the passed probability. Estimated filter size is set to zero, which means
// means more reallocations are done when building the filter. // more reallocations are done when building the filter.
func WithRandomKeyP(p uint8) *GCSBuilder { func WithRandomKeyP(p uint8) *GCSBuilder {
return WithRandomKeyPN(p, 0) return WithRandomKeyPN(p, 0)
} }
// WithRandomKey creates a GCSBuilder with a cryptographically random // WithRandomKey creates a GCSBuilder with a cryptographically random key.
// key. Probability is set to 20 (2^-20 collision probability). Estimated // Probability is set to 20 (2^-20 collision probability). Estimated filter
// filter size is set to zero, which means more reallocations are done when // size is set to zero, which means more reallocations are done when
// building the filter. // building the filter.
func WithRandomKey() *GCSBuilder { func WithRandomKey() *GCSBuilder {
return WithRandomKeyPN(DefaultP, 0) return WithRandomKeyPN(DefaultP, 0)
} }
// BuildBasicFilter builds a basic GCS filter from a block. // BuildBasicFilter builds a basic GCS filter from a block. A basic GCS filter
// will contain all the previous outpoints spent within a block, as well as the
// data pushes within all the outputs created within a block.
func BuildBasicFilter(block *wire.MsgBlock) (*gcs.Filter, error) { func BuildBasicFilter(block *wire.MsgBlock) (*gcs.Filter, error) {
blockHash := block.BlockHash() blockHash := block.BlockHash()
b := WithKeyHash(&blockHash) b := WithKeyHash(&blockHash)
// If the filter had an issue with the specified key, then we force it
// to bubble up here by calling the Key() function.
_, err := b.Key() _, err := b.Key()
if err != nil { if err != nil {
return nil, err return nil, err
} }
// In order to build a basic filter, we'll range over the entire block,
// adding the outpoint data as well as the data pushes within the
// pkScript.
for i, tx := range block.Transactions { for i, tx := range block.Transactions {
// Skip the inputs for the coinbase transaction // Skip the inputs for the coinbase transaction
if i != 0 { if i != 0 {
// Each each txin, we'll add a serialized version of
// the txid:index to the filters data slices.
for _, txIn := range tx.TxIn { for _, txIn := range tx.TxIn {
b.AddOutPoint(txIn.PreviousOutPoint) b.AddOutPoint(txIn.PreviousOutPoint)
} }
} }
// For each output in a transaction, we'll add each of the
// individual data pushes within the script.
for _, txOut := range tx.TxOut { for _, txOut := range tx.TxOut {
b.AddScript(txOut.PkScript) b.AddScript(txOut.PkScript)
} }
} }
return b.Build() return b.Build()
} }
// BuildExtFilter builds an extended GCS filter from a block. // BuildExtFilter builds an extended GCS filter from a block. An extended
// filter supplements a regular basic filter by include all the _witness_ data
// found within a block. This includes all the data pushes within any signature
// scripts as well as each element of an input's witness stack. Additionally,
// the _hashes_ of each transaction are also inserted into the filter.
func BuildExtFilter(block *wire.MsgBlock) (*gcs.Filter, error) { func BuildExtFilter(block *wire.MsgBlock) (*gcs.Filter, error) {
blockHash := block.BlockHash() blockHash := block.BlockHash()
b := WithKeyHash(&blockHash) b := WithKeyHash(&blockHash)
@ -312,6 +332,7 @@ func BuildExtFilter(block *wire.MsgBlock) (*gcs.Filter, error) {
} }
} }
} }
return b.Build() return b.Build()
} }
@ -326,8 +347,14 @@ func GetFilterHash(filter *gcs.Filter) chainhash.Hash {
func MakeHeaderForFilter(filter *gcs.Filter, prevHeader chainhash.Hash) chainhash.Hash { func MakeHeaderForFilter(filter *gcs.Filter, prevHeader chainhash.Hash) chainhash.Hash {
filterTip := make([]byte, 2*chainhash.HashSize) filterTip := make([]byte, 2*chainhash.HashSize)
filterHash := GetFilterHash(filter) filterHash := GetFilterHash(filter)
// In the buffer we created above we'll compute hash || prevHash as an
// intermediate value.
copy(filterTip, filterHash[:]) copy(filterTip, filterHash[:])
copy(filterTip[chainhash.HashSize:], prevHeader[:]) copy(filterTip[chainhash.HashSize:], prevHeader[:])
// The final filter hash is the double-sha256 of the hash computed
// above.
hash1 := chainhash.HashH(filterTip) hash1 := chainhash.HashH(filterTip)
return chainhash.HashH(hash1[:]) return chainhash.HashH(hash1[:])
} }

View file

@ -30,19 +30,18 @@ var (
) )
const ( const (
//KeySize is the size of the byte array required for key material for // KeySize is the size of the byte array required for key material for
// the SipHash keyed hash function. // the SipHash keyed hash function.
KeySize = 16 KeySize = 16
) )
// Filter describes an immutable filter that can be built from // Filter describes an immutable filter that can be built from a set of data
// a set of data elements, serialized, deserialized, and queried // elements, serialized, deserialized, and queried in a thread-safe manner. The
// in a thread-safe manner. The serialized form is compressed as // serialized form is compressed as a Golomb Coded Set (GCS), but does not
// a Golomb Coded Set (GCS), but does not include N or P to allow // include N or P to allow the user to encode the metadata separately if
// the user to encode the metadata separately if necessary. The // necessary. The hash function used is SipHash, a keyed function; the key used
// hash function used is SipHash, a keyed function; the key used // in building the filter is required in order to match filter values and is
// in building the filter is required in order to match filter // not included in the serialized form.
// values and is not included in the serialized form.
type Filter struct { type Filter struct {
n uint32 n uint32
p uint8 p uint8
@ -54,9 +53,7 @@ type Filter struct {
// BuildGCSFilter builds a new GCS filter with the collision probability of // BuildGCSFilter builds a new GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of // `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set. // the set.
func BuildGCSFilter(P uint8, key [KeySize]byte, func BuildGCSFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
data [][]byte) (*Filter, error) {
// Some initial parameter checks: make sure we have data from which to // Some initial parameter checks: make sure we have data from which to
// build the filter, and make sure our parameters will fit the hash // build the filter, and make sure our parameters will fit the hash
// function we're using. // function we're using.
@ -97,10 +94,12 @@ func BuildGCSFilter(P uint8, key [KeySize]byte,
// Calculate the difference between this value and the last, // Calculate the difference between this value and the last,
// modulo P. // modulo P.
remainder = (v - lastValue) % f.modulusP remainder = (v - lastValue) % f.modulusP
// Calculate the difference between this value and the last, // Calculate the difference between this value and the last,
// divided by P. // divided by P.
value = (v - lastValue - remainder) / f.modulusP value = (v - lastValue - remainder) / f.modulusP
lastValue = v lastValue = v
// Write the P multiple into the bitstream in unary; the // Write the P multiple into the bitstream in unary; the
// average should be around 1 (2 bits - 0b10). // average should be around 1 (2 bits - 0b10).
for value > 0 { for value > 0 {
@ -108,6 +107,7 @@ func BuildGCSFilter(P uint8, key [KeySize]byte,
value-- value--
} }
b.WriteBit(false) b.WriteBit(false)
// Write the remainder as a big-endian integer with enough bits // Write the remainder as a big-endian integer with enough bits
// to represent the appropriate collision probability. // to represent the appropriate collision probability.
b.WriteBits(remainder, int(f.p)) b.WriteBits(remainder, int(f.p))
@ -115,11 +115,12 @@ func BuildGCSFilter(P uint8, key [KeySize]byte,
// Copy the bitstream into the filter object and return the object. // Copy the bitstream into the filter object and return the object.
f.filterData = b.Bytes() f.filterData = b.Bytes()
return &f, nil return &f, nil
} }
// FromBytes deserializes a GCS filter from a known N, P, and serialized // FromBytes deserializes a GCS filter from a known N, P, and serialized filter
// filter as returned by Bytes(). // as returned by Bytes().
func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) { func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {
// Basic sanity check. // Basic sanity check.
@ -138,6 +139,7 @@ func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {
// Copy the filter. // Copy the filter.
f.filterData = make([]byte, len(d)) f.filterData = make([]byte, len(d))
copy(f.filterData, d) copy(f.filterData, d)
return f, nil return f, nil
} }
@ -206,8 +208,8 @@ func (f *Filter) N() uint32 {
return f.n return f.n
} }
// Match checks whether a []byte value is likely (within collision // Match checks whether a []byte value is likely (within collision probability)
// probability) to be a member of the set represented by the filter. // to be a member of the set represented by the filter.
func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) { func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {
// Create a filter bitstream. // Create a filter bitstream.
@ -220,8 +222,9 @@ func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {
// Go through the search filter and look for the desired value. // Go through the search filter and look for the desired value.
var lastValue uint64 var lastValue uint64
for lastValue < term { for lastValue < term {
// Read the difference between previous and new value
// from bitstream. // Read the difference between previous and new value from
// bitstream.
value, err := f.readFullUint64(b) value, err := f.readFullUint64(b)
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
@ -229,19 +232,22 @@ func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {
} }
return false, err return false, err
} }
// Add the previous value to it. // Add the previous value to it.
value += lastValue value += lastValue
if value == term { if value == term {
return true, nil return true, nil
} }
lastValue = value lastValue = value
} }
return false, nil return false, nil
} }
// MatchAny returns checks whether any []byte value is likely (within // MatchAny returns checks whether any []byte value is likely (within collision
// collision probability) to be a member of the set represented by the // probability) to be a member of the set represented by the filter faster than
// filter faster than calling Match() for each value individually. // calling Match() for each value individually.
func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) { func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {
// Basic sanity check. // Basic sanity check.
@ -262,7 +268,8 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {
sort.Sort(values) sort.Sort(values)
// Zip down the filters, comparing values until we either run out of // Zip down the filters, comparing values until we either run out of
// values to compare in one of the filters or we reach a matching value. // values to compare in one of the filters or we reach a matching
// value.
var lastValue1, lastValue2 uint64 var lastValue1, lastValue2 uint64
lastValue2 = values[0] lastValue2 = values[0]
i := 1 i := 1
@ -292,13 +299,14 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {
lastValue1 += value lastValue1 += value
} }
} }
// If we've made it this far, an element matched between filters so
// we return true. // If we've made it this far, an element matched between filters so we
// return true.
return true, nil return true, nil
} }
// readFullUint64 reads a value represented by the sum of a unary multiple // readFullUint64 reads a value represented by the sum of a unary multiple of
// of the filter's P modulus (`2**P`) and a big-endian P-bit remainder. // the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *Filter) readFullUint64(b *bstream.BStream) (uint64, error) { func (f *Filter) readFullUint64(b *bstream.BStream) (uint64, error) {
var v uint64 var v uint64

View file

@ -22,8 +22,8 @@ var (
P = uint8(20) P = uint8(20)
// Filters are conserved between tests but we must define with an // Filters are conserved between tests but we must define with an
// interface which functions we're testing because the gcsFilter // interface which functions we're testing because the gcsFilter type
// type isn't exported // isn't exported
filter, filter2, filter3, filter4, filter5 *gcs.Filter filter, filter2, filter3, filter4, filter5 *gcs.Filter
// We need to use the same key for building and querying the filters // We need to use the same key for building and querying the filters
@ -73,8 +73,8 @@ var (
) )
// TestGCSFilterBuild builds a test filter with a randomized key. For Bitcoin // TestGCSFilterBuild builds a test filter with a randomized key. For Bitcoin
// use, deterministic filter generation is desired. Therefore, a // use, deterministic filter generation is desired. Therefore, a key that's
// key that's derived deterministically would be required. // derived deterministically would be required.
func TestGCSFilterBuild(t *testing.T) { func TestGCSFilterBuild(t *testing.T) {
for i := 0; i < gcs.KeySize; i += 4 { for i := 0; i < gcs.KeySize; i += 4 {
binary.BigEndian.PutUint32(key[i:], rand.Uint32()) binary.BigEndian.PutUint32(key[i:], rand.Uint32())
@ -105,8 +105,8 @@ func TestGCSFilterCopy(t *testing.T) {
} }
} }
// TestGCSFilterMetadata checks that the filter metadata is built and // TestGCSFilterMetadata checks that the filter metadata is built and copied
// copied correctly. // correctly.
func TestGCSFilterMetadata(t *testing.T) { func TestGCSFilterMetadata(t *testing.T) {
if filter.P() != P { if filter.P() != P {
t.Fatal("P not correctly stored in filter metadata") t.Fatal("P not correctly stored in filter metadata")
@ -213,8 +213,8 @@ func TestGCSFilterMatch(t *testing.T) {
} }
} }
// TestGCSFilterMatchAny checks that both the built and copied filters match // TestGCSFilterMatchAny checks that both the built and copied filters match a
// a list correctly, logging any false positives without failing on them. // list correctly, logging any false positives without failing on them.
func TestGCSFilterMatchAny(t *testing.T) { func TestGCSFilterMatchAny(t *testing.T) {
match, err := filter.MatchAny(key, contents2) match, err := filter.MatchAny(key, contents2)
if err != nil { if err != nil {

View file

@ -41,7 +41,8 @@ func BenchmarkGCSFilterMatch(b *testing.B) {
} }
} }
// BenchmarkGCSFilterMatchAny benchmarks querying a filter for a list of values. // BenchmarkGCSFilterMatchAny benchmarks querying a filter for a list of
// values.
func BenchmarkGCSFilterMatchAny(b *testing.B) { func BenchmarkGCSFilterMatchAny(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
filter.MatchAny(key, contents2) filter.MatchAny(key, contents2)

View file

@ -5,8 +5,8 @@
package gcs package gcs
// uint64slice is a package-local utility class that allows us to use Go's // uint64slice is a package-local utility class that allows us to use Go's sort
// sort package to sort a []uint64 by implementing sort.Interface. // package to sort a []uint64 by implementing sort.Interface.
type uint64Slice []uint64 type uint64Slice []uint64
// Len returns the length of the slice. // Len returns the length of the slice.
@ -14,8 +14,8 @@ func (p uint64Slice) Len() int {
return len(p) return len(p)
} }
// Less returns true when the ith element is smaller than the jth element // Less returns true when the ith element is smaller than the jth element of
// of the slice, and returns false otherwise. // the slice, and returns false otherwise.
func (p uint64Slice) Less(i, j int) bool { func (p uint64Slice) Less(i, j int) bool {
return p[i] < p[j] return p[i] < p[j]
} }