LBRYCommunity/lbcutil
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

gcs/builder: Deduplicate items before creating block filters.

Browse source
This commit is contained in:
Jim Posen 2018-01-22 22:05:04 -08:00 committed by Olaoluwa Osuntokun
parent cbc2d0fee6
commit ad0070fa44
2 changed files with 27 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
gcs/builder/builder.go
View file

@ -20,9 +20,12 @@ const DefaultP = 20
// GCSBuilder is a utility class that makes building GCS filters convenient. // GCSBuilder is a utility class that makes building GCS filters convenient.
type GCSBuilder struct { type GCSBuilder struct {
p uint8 p uint8
key [gcs.KeySize]byte key [gcs.KeySize]byte
data [][]byte
// data is a set of entries represented as strings. This is done to
// deduplicate items as they are added.
data map[string]struct{}
err error err error
} }
@ -125,8 +128,8 @@ func (b *GCSBuilder) Preallocate(n uint32) *GCSBuilder {
return b return b
} }
if len(b.data) == 0 { if b.data == nil {
b.data = make([][]byte, 0, n) b.data = make(map[string]struct{}, n)
} }
return b return b
@ -140,7 +143,7 @@ func (b *GCSBuilder) AddEntry(data []byte) *GCSBuilder {
return b return b
} }
b.data = append(b.data, data) b.data[string(data)] = struct{}{}
return b return b
} }
@ -217,7 +220,12 @@ func (b *GCSBuilder) Build() (*gcs.Filter, error) {
return nil, b.err return nil, b.err
} }
return gcs.BuildGCSFilter(b.p, b.key, b.data) dataSlice := make([][]byte, 0, len(b.data))
for item := range b.data {
dataSlice = append(dataSlice, []byte(item))
}
return gcs.BuildGCSFilter(b.p, b.key, dataSlice)
} }
// WithKeyPN creates a GCSBuilder with specified key and the passed probability // WithKeyPN creates a GCSBuilder with specified key and the passed probability

12
gcs/builder/builder_test.go
View file

@ -286,4 +286,16 @@ func BuilderTest(b *builder.GCSBuilder, hash *chainhash.Hash, p uint8,
if !match { if !match {
t.Fatal("Filter didn't match when it should have!") t.Fatal("Filter didn't match when it should have!")
} }
// Check that adding duplicate items does not increase filter size.
originalSize := f.N()
b.AddScript(addrBytes)
b.AddWitness(witness)
f, err = b.Build()
if err != nil {
t.Fatalf("Filter build failed: %s", err.Error())
}
if f.N() != originalSize {
t.Fatal("Filter size increased with duplicate items")
}
} }