gcs: update modulo algorithm to fast range per gmaxwell's suggestion

The BIP will now specify that the fast range described at the URL
below is used instead of modulo:

https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
This commit is contained in:
Alex 2017-09-25 15:13:57 -06:00 committed by Olaoluwa Osuntokun
parent 0878c162dd
commit dabaf053db

View file

@ -79,11 +79,31 @@ func BuildGCSFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error)
// Build the filter. // Build the filter.
values := make(uint64Slice, 0, len(data)) values := make(uint64Slice, 0, len(data))
b := bstream.NewBStreamWriter(0) b := bstream.NewBStreamWriter(0)
// Insert the hash (fast-ranged over a space of N*P) of each data
// Insert the hash (modulo N*P) of each data element into a slice and // element into a slice and sort the slice. This can be greatly
// sort the slice. // optimized with native 128-bit multiplication, but we're going to be
// fully portable for now.
var v, vhi, vlo, nphi, nplo, vnphi, vnpmid, npvmid, vnplo, carry uint64
// First, we cache the high and low bits of modulusNP for the
// multiplication of 2 64-bit integers into a 128-bit integer.
nphi = f.modulusNP >> 32
nplo = uint64(uint32(f.modulusNP))
for _, d := range data { for _, d := range data {
v := siphash.Sum64(d, &key) % f.modulusNP // For each datum, we assign the initial hash to a uint64.
v = siphash.Sum64(d, &key)
// Then, we split it into high bits and low bits.
vhi = v >> 32
vlo = uint64(uint32(v))
// Then, we distribute multiplication over each part.
vnphi = vhi * nphi
vnpmid = vhi * nplo
npvmid = nphi * vlo
vnplo = vlo * nplo
// We calculate the carry bit.
carry = (uint64(uint32(vnpmid)) + uint64(uint32(npvmid)) +
(vnplo >> 32)) >> 32
// Last, we add the high bits, the middle bits, and the carry.
v = vnphi + (vnpmid >> 32) + (npvmid >> 32) + carry
values = append(values, v) values = append(values, v)
} }
sort.Sort(values) sort.Sort(values)
@ -217,8 +237,25 @@ func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {
filterData := f.Bytes() filterData := f.Bytes()
b := bstream.NewBStreamReader(filterData) b := bstream.NewBStreamReader(filterData)
// Hash our search term with the same parameters as the filter. // We take the high and low bits of modulusNP for the multiplication
term := siphash.Sum64(data, &key) % f.modulusNP // of 2 64-bit integers into a 128-bit integer.
nphi := f.modulusNP >> 32
nplo := uint64(uint32(f.modulusNP))
// Then we hash our search term with the same parameters as the filter.
term := siphash.Sum64(data, &key)
// Then, we split it into high bits and low bits.
vhi := term >> 32
vlo := uint64(uint32(term))
// Then, we distribute multiplication over each part.
vnphi := vhi * nphi
vnpmid := vhi * nplo
npvmid := nphi * vlo
vnplo := vlo * nplo
// We calculate the carry bit.
carry := (uint64(uint32(vnpmid)) + uint64(uint32(npvmid)) +
(vnplo >> 32)) >> 32
// Last, we add the high bits, the middle bits, and the carry.
term = vnphi + (vnpmid >> 32) + (npvmid >> 32) + carry
// Go through the search filter and look for the desired value. // Go through the search filter and look for the desired value.
var lastValue uint64 var lastValue uint64
@ -262,8 +299,27 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {
// Create an uncompressed filter of the search values. // Create an uncompressed filter of the search values.
values := make(uint64Slice, 0, len(data)) values := make(uint64Slice, 0, len(data))
var v, vhi, vlo, nphi, nplo, vnphi, vnpmid, npvmid, vnplo, carry uint64
// First, we cache the high and low bits of modulusNP for the
// multiplication of 2 64-bit integers into a 128-bit integer.
nphi = f.modulusNP >> 32
nplo = uint64(uint32(f.modulusNP))
for _, d := range data { for _, d := range data {
v := siphash.Sum64(d, &key) % f.modulusNP // For each datum, we assign the initial hash to a uint64.
v = siphash.Sum64(d, &key)
// Then, we split it into high bits and low bits.
vhi = v >> 32
vlo = uint64(uint32(v))
// Then, we distribute multiplication over each part.
vnphi = vhi * nphi
vnpmid = vhi * nplo
npvmid = nphi * vlo
vnplo = vlo * nplo
// We calculate the carry bit.
carry = (uint64(uint32(vnpmid)) + uint64(uint32(npvmid)) +
(vnplo >> 32)) >> 32
// Last, we add the high bits, the middle bits, and the carry.
v = vnphi + (vnpmid >> 32) + (npvmid >> 32) + carry
values = append(values, v) values = append(values, v)
} }
sort.Sort(values) sort.Sort(values)