lbcutil/gcs/gcs_test.go

// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package gcs_test

import (
	"bytes"
	"encoding/binary"
	"math/rand"
	"testing"

	"github.com/lbryio/lbcutil/gcs"
)

var (
	// No need to allocate an err variable in every test
	err error

	// Collision probability for the tests (1/2**19)
	P = uint8(19)

	// Modulus value for the tests.
	M uint64 = 784931

	// Filters are conserved between tests but we must define with an
	// interface which functions we're testing because the gcsFilter type
	// isn't exported
	filter, filter2, filter3 *gcs.Filter

	// We need to use the same key for building and querying the filters
	key [gcs.KeySize]byte

	// List of values for building a filter
	contents = [][]byte{
		[]byte("Alex"),
		[]byte("Bob"),
		[]byte("Charlie"),
		[]byte("Dick"),
		[]byte("Ed"),
		[]byte("Frank"),
		[]byte("George"),
		[]byte("Harry"),
		[]byte("Ilya"),
		[]byte("John"),
		[]byte("Kevin"),
		[]byte("Larry"),
		[]byte("Michael"),
		[]byte("Nate"),
		[]byte("Owen"),
		[]byte("Paul"),
		[]byte("Quentin"),
	}

	// List of values for querying a filter using MatchAny()
	contents2 = [][]byte{
		[]byte("Alice"),
		[]byte("Betty"),
		[]byte("Charmaine"),
		[]byte("Donna"),
		[]byte("Edith"),
		[]byte("Faina"),
		[]byte("Georgia"),
		[]byte("Hannah"),
		[]byte("Ilsbeth"),
		[]byte("Jennifer"),
		[]byte("Kayla"),
		[]byte("Lena"),
		[]byte("Michelle"),
		[]byte("Natalie"),
		[]byte("Ophelia"),
		[]byte("Peggy"),
		[]byte("Queenie"),
	}
)

// TestGCSFilterBuild builds a test filter with a randomized key. For Bitcoin
// use, deterministic filter generation is desired. Therefore, a key that's
// derived deterministically would be required.
func TestGCSFilterBuild(t *testing.T) {
	for i := 0; i < gcs.KeySize; i += 4 {
		binary.BigEndian.PutUint32(key[i:], rand.Uint32())
	}
	filter, err = gcs.BuildGCSFilter(P, M, key, contents)
	if err != nil {
		t.Fatalf("Filter build failed: %s", err.Error())
	}
}

// TestGCSMatchZeroHash ensures that Match and MatchAny properly match an item
// if it's hash after the reduction is zero. This is accomplished by brute
// forcing a specific target whose hash is zero given a certain (P, M, key,
// len(elements)) combination. In this case, P and M are the default, key was
// chosen randomly, and len(elements) is 13. The target 4-byte value of 16060032
// is the first such 32-bit value, thus we use the number 0-11 as the other
// elements in the filter since we know they won't collide. We test both the
// positive and negative cases, when the zero hash item is in the filter and
// when it is excluded. In the negative case, the 32-bit value of 12 is added to
// the filter instead of the target.
func TestGCSMatchZeroHash(t *testing.T) {
	t.Run("include zero", func(t *testing.T) {
		testGCSMatchZeroHash(t, true)
	})
	t.Run("exclude zero", func(t *testing.T) {
		testGCSMatchZeroHash(t, false)
	})
}

func testGCSMatchZeroHash(t *testing.T, includeZeroHash bool) {
	key := [gcs.KeySize]byte{
		0x25, 0x28, 0x0d, 0x25, 0x26, 0xe1, 0xd3, 0xc7,
		0xa5, 0x71, 0x85, 0x34, 0x92, 0xa5, 0x7e, 0x68,
	}

	// Construct the target data to match, whose hash is zero after applying
	// the reduction with the parameters in the test.
	target := make([]byte, 4)
	binary.BigEndian.PutUint32(target, 16060032)

	// Construct the set of 13 items including the target, using the 32-bit
	// values of 0 through 11 as the first 12 items. We known none of these
	// hash to zero since the brute force ended well beyond them.
	elements := make([][]byte, 0, 13)
	for i := 0; i < 12; i++ {
		data := make([]byte, 4)
		binary.BigEndian.PutUint32(data, uint32(i))
		elements = append(elements, data)
	}

	// If the filter should include the zero hash element, add the target
	// which we know hashes to zero. Otherwise add 32-bit value of 12 which
	// we know does not hash to zero.
	if includeZeroHash {
		elements = append(elements, target)
	} else {
		data := make([]byte, 4)
		binary.BigEndian.PutUint32(data, 12)
		elements = append(elements, data)
	}

	filter, err := gcs.BuildGCSFilter(P, M, key, elements)
	if err != nil {
		t.Fatalf("unable to build filter: %v", err)
	}

	match, err := filter.Match(key, target)
	if err != nil {
		t.Fatalf("unable to match: %v", err)
	}

	// We should only get a match iff the target was included.
	if match != includeZeroHash {
		t.Fatalf("expected match from Match: %t, got %t",
			includeZeroHash, match)
	}

	match, err = filter.MatchAny(key, [][]byte{target})
	if err != nil {
		t.Fatalf("unable to match any: %v", err)
	}

	// We should only get a match iff the target was included.
	if match != includeZeroHash {
		t.Fatalf("expected match from MatchAny: %t, got %t",
			includeZeroHash, match)
	}
}

// TestGCSFilterCopy deserializes and serializes a filter to create a copy.
func TestGCSFilterCopy(t *testing.T) {
	serialized2, err := filter.Bytes()
	if err != nil {
		t.Fatalf("Filter Bytes() failed: %v", err)
	}
	filter2, err = gcs.FromBytes(filter.N(), P, M, serialized2)
	if err != nil {
		t.Fatalf("Filter copy failed: %s", err.Error())
	}
	serialized3, err := filter.NBytes()
	if err != nil {
		t.Fatalf("Filter NBytes() failed: %v", err)
	}
	filter3, err = gcs.FromNBytes(filter.P(), M, serialized3)
	if err != nil {
		t.Fatalf("Filter copy failed: %s", err.Error())
	}
}

// TestGCSFilterMetadata checks that the filter metadata is built and copied
// correctly.
func TestGCSFilterMetadata(t *testing.T) {
	if filter.P() != P {
		t.Fatal("P not correctly stored in filter metadata")
	}
	if filter.N() != uint32(len(contents)) {
		t.Fatal("N not correctly stored in filter metadata")
	}
	if filter.P() != filter2.P() {
		t.Fatal("P doesn't match between copied filters")
	}
	if filter.P() != filter3.P() {
		t.Fatal("P doesn't match between copied filters")
	}
	if filter.N() != filter2.N() {
		t.Fatal("N doesn't match between copied filters")
	}
	if filter.N() != filter3.N() {
		t.Fatal("N doesn't match between copied filters")
	}
	serialized, err := filter.Bytes()
	if err != nil {
		t.Fatalf("Filter Bytes() failed: %v", err)
	}
	serialized2, err := filter2.Bytes()
	if err != nil {
		t.Fatalf("Filter Bytes() failed: %v", err)
	}
	if !bytes.Equal(serialized, serialized2) {
		t.Fatal("Bytes don't match between copied filters")
	}
	serialized3, err := filter3.Bytes()
	if err != nil {
		t.Fatalf("Filter Bytes() failed: %v", err)
	}
	if !bytes.Equal(serialized, serialized3) {
		t.Fatal("Bytes don't match between copied filters")
	}
	serialized4, err := filter3.Bytes()
	if err != nil {
		t.Fatalf("Filter Bytes() failed: %v", err)
	}
	if !bytes.Equal(serialized, serialized4) {
		t.Fatal("Bytes don't match between copied filters")
	}
}

// TestGCSFilterMatch checks that both the built and copied filters match
// correctly, logging any false positives without failing on them.
func TestGCSFilterMatch(t *testing.T) {
	match, err := filter.Match(key, []byte("Nate"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if !match {
		t.Fatal("Filter didn't match when it should have!")
	}
	match, err = filter2.Match(key, []byte("Nate"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if !match {
		t.Fatal("Filter didn't match when it should have!")
	}
	match, err = filter.Match(key, []byte("Quentin"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if !match {
		t.Fatal("Filter didn't match when it should have!")
	}
	match, err = filter2.Match(key, []byte("Quentin"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if !match {
		t.Fatal("Filter didn't match when it should have!")
	}
	match, err = filter.Match(key, []byte("Nates"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if match {
		t.Logf("False positive match, should be 1 in 2**%d!", P)
	}
	match, err = filter2.Match(key, []byte("Nates"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if match {
		t.Logf("False positive match, should be 1 in 2**%d!", P)
	}
	match, err = filter.Match(key, []byte("Quentins"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if match {
		t.Logf("False positive match, should be 1 in 2**%d!", P)
	}
	match, err = filter2.Match(key, []byte("Quentins"))
	if err != nil {
		t.Fatalf("Filter match failed: %s", err.Error())
	}
	if match {
		t.Logf("False positive match, should be 1 in 2**%d!", P)
	}
}

// AnyMatcher is the function signature of our matching algorithms.
type AnyMatcher func(key [gcs.KeySize]byte, data [][]byte) (bool, error)

// TestGCSFilterMatchAnySuite checks that all of our matching algorithms
// properly match a list correctly when using built or copied filters, logging
// any false positives without failing on them.
func TestGCSFilterMatchAnySuite(t *testing.T) {
	funcs := []struct {
		name     string
		matchAny func(*gcs.Filter) AnyMatcher
	}{
		{
			"default",
			func(f *gcs.Filter) AnyMatcher {
				return f.MatchAny
			},
		},
		{
			"hash",
			func(f *gcs.Filter) AnyMatcher {
				return f.HashMatchAny
			},
		},
		{
			"zip",
			func(f *gcs.Filter) AnyMatcher {
				return f.ZipMatchAny
			},
		},
	}

	for _, test := range funcs {
		test := test

		t.Run(test.name, func(t *testing.T) {
			contentsCopy := make([][]byte, len(contents2))
			copy(contentsCopy, contents2)

			match, err := test.matchAny(filter)(key, contentsCopy)
			if err != nil {
				t.Fatalf("Filter match any failed: %s", err.Error())
			}
			if match {
				t.Logf("False positive match, should be 1 in 2**%d!", P)
			}
			match, err = test.matchAny(filter2)(key, contentsCopy)
			if err != nil {
				t.Fatalf("Filter match any failed: %s", err.Error())
			}
			if match {
				t.Logf("False positive match, should be 1 in 2**%d!", P)
			}
			contentsCopy = append(contentsCopy, []byte("Nate"))
			match, err = test.matchAny(filter)(key, contentsCopy)
			if err != nil {
				t.Fatalf("Filter match any failed: %s", err.Error())
			}
			if !match {
				t.Fatal("Filter didn't match any when it should have!")
			}
			match, err = test.matchAny(filter2)(key, contentsCopy)
			if err != nil {
				t.Fatalf("Filter match any failed: %s", err.Error())
			}
			if !match {
				t.Fatal("Filter didn't match any when it should have!")
			}
		})
	}
}