Merge #14074: Use std::unordered_set instead of set in blockfilter interface

fef5adcc33 blockfilter: Use unordered_set instead of set in blockfilter. (Jim Posen)
4fb789e9b2 Extract CSipHasher to it's own file in crypto/ directory. (Jim Posen)

Pull request description:

  Use `std::unordered_set` (hash set) instead of `std::set` (tree set) in blockfilter interface, as suggested by @ryanofsky in #12254. This may result in a very minor speedup, but I haven't measured.

  This moves `CSipHasher` to it's own file `crypto/siphash.h`, so that it can be used in the libbitcoin_util library without including `hash.{h,cpp}`. I'm open to other suggestions on solving this issue if people would prefer to leave CSipHasher where it is.

Tree-SHA512: 593d1abda771e45f2860d5334272980d20df0b81925a402bb9ee875e17595c2517c0d8ac9c579218b84bbf66e15b49418241c1fe9f9265719bcd2377b0cd0d88

src/Makefile.am

@@ -183,6 +183,7 @@ BITCOIN_CORE_H = \
   txmempool.h \
   ui_interface.h \
   undo.h \
+  util/bytevectorhash.h \
   util/system.h \
   util/memory.h \
   util/moneystr.h \
@@ -321,7 +322,9 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \
   crypto/sha256.cpp \
   crypto/sha256.h \
   crypto/sha512.cpp \
-  crypto/sha512.h
+  crypto/sha512.h \
+  crypto/siphash.cpp \
+  crypto/siphash.h
 
 if USE_ASM
 crypto_libbitcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp
@@ -427,6 +430,7 @@ libbitcoin_util_a_SOURCES = \
   support/cleanse.cpp \
   sync.cpp \
   threadinterrupt.cpp \
+  util/bytevectorhash.cpp \
   util/system.cpp \
   util/moneystr.cpp \
   util/strencodings.cpp \

src/bench/block_assemble.cpp

@@ -7,6 +7,7 @@
 #include <coins.h>
 #include <consensus/merkle.h>
 #include <consensus/validation.h>
+#include <crypto/sha256.h>
 #include <miner.h>
 #include <policy/policy.h>
 #include <pow.h>

src/bench/crypto_hash.cpp

@@ -14,6 +14,7 @@
 #include <crypto/sha1.h>
 #include <crypto/sha256.h>
 #include <crypto/sha512.h>
+#include <crypto/siphash.h>
 
 /* Number of bytes to hash per iteration */
 static const uint64_t BUFFER_SIZE = 1000*1000;

src/blockencodings.cpp

@@ -6,7 +6,8 @@
 #include <consensus/consensus.h>
 #include <consensus/validation.h>
 #include <chainparams.h>
-#include <hash.h>
+#include <crypto/sha256.h>
+#include <crypto/siphash.h>
 #include <random.h>
 #include <streams.h>
 #include <txmempool.h>

src/blockfilter.cpp

@@ -3,6 +3,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <blockfilter.h>
+#include <crypto/siphash.h>
 #include <hash.h>
 #include <primitives/transaction.h>
 #include <script/script.h>

src/blockfilter.h

@@ -5,14 +5,15 @@
 #ifndef BITCOIN_BLOCKFILTER_H
 #define BITCOIN_BLOCKFILTER_H
 
-#include <set>
 #include <stdint.h>
+#include <unordered_set>
 #include <vector>
 
 #include <primitives/block.h>
 #include <serialize.h>
 #include <uint256.h>
 #include <undo.h>
+#include <util/bytevectorhash.h>
 
 /**
  * This implements a Golomb-coded set as defined in BIP 158. It is a
@@ -22,7 +23,7 @@ class GCSFilter
 {
 public:
     typedef std::vector<unsigned char> Element;
-    typedef std::set<Element> ElementSet;
+    typedef std::unordered_set<Element, ByteVectorHash> ElementSet;
 
 private:
     uint64_t m_siphash_k0;

src/coins.cpp

@@ -6,6 +6,7 @@
 
 #include <consensus/consensus.h>
 #include <random.h>
+#include <version.h>
 
 bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; }
 uint256 CCoinsView::GetBestBlock() const { return uint256(); }

src/coins.h

@@ -9,7 +9,7 @@
 #include <primitives/transaction.h>
 #include <compressor.h>
 #include <core_memusage.h>
-#include <hash.h>
+#include <crypto/siphash.h>
 #include <memusage.h>
 #include <serialize.h>
 #include <uint256.h>

src/crypto/siphash.cpp

@@ -0,0 +1,173 @@
+// Copyright (c) 2016-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <crypto/siphash.h>
+
+#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
+
+#define SIPROUND do { \
+    v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \
+    v0 = ROTL(v0, 32); \
+    v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \
+    v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \
+    v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \
+    v2 = ROTL(v2, 32); \
+} while (0)
+
+CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
+{
+    v[0] = 0x736f6d6570736575ULL ^ k0;
+    v[1] = 0x646f72616e646f6dULL ^ k1;
+    v[2] = 0x6c7967656e657261ULL ^ k0;
+    v[3] = 0x7465646279746573ULL ^ k1;
+    count = 0;
+    tmp = 0;
+}
+
+CSipHasher& CSipHasher::Write(uint64_t data)
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+
+    assert(count % 8 == 0);
+
+    v3 ^= data;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= data;
+
+    v[0] = v0;
+    v[1] = v1;
+    v[2] = v2;
+    v[3] = v3;
+
+    count += 8;
+    return *this;
+}
+
+CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size)
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+    uint64_t t = tmp;
+    int c = count;
+
+    while (size--) {
+        t |= ((uint64_t)(*(data++))) << (8 * (c % 8));
+        c++;
+        if ((c & 7) == 0) {
+            v3 ^= t;
+            SIPROUND;
+            SIPROUND;
+            v0 ^= t;
+            t = 0;
+        }
+    }
+
+    v[0] = v0;
+    v[1] = v1;
+    v[2] = v2;
+    v[3] = v3;
+    count = c;
+    tmp = t;
+
+    return *this;
+}
+
+uint64_t CSipHasher::Finalize() const
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+
+    uint64_t t = tmp | (((uint64_t)count) << 56);
+
+    v3 ^= t;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= t;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}
+
+uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val)
+{
+    /* Specialized implementation for efficiency */
+    uint64_t d = val.GetUint64(0);
+
+    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
+    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
+    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
+    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
+
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(1);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(2);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(3);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    v3 ^= ((uint64_t)4) << 59;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= ((uint64_t)4) << 59;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}
+
+uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra)
+{
+    /* Specialized implementation for efficiency */
+    uint64_t d = val.GetUint64(0);
+
+    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
+    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
+    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
+    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
+
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(1);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(2);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(3);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = (((uint64_t)36) << 56) | extra;
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}

src/crypto/siphash.h

@@ -0,0 +1,47 @@
+// Copyright (c) 2016-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_CRYPTO_SIPHASH_H
+#define BITCOIN_CRYPTO_SIPHASH_H
+
+#include <stdint.h>
+
+#include <uint256.h>
+
+/** SipHash-2-4 */
+class CSipHasher
+{
+private:
+    uint64_t v[4];
+    uint64_t tmp;
+    int count;
+
+public:
+    /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */
+    CSipHasher(uint64_t k0, uint64_t k1);
+    /** Hash a 64-bit integer worth of data
+     *  It is treated as if this was the little-endian interpretation of 8 bytes.
+     *  This function can only be used when a multiple of 8 bytes have been written so far.
+     */
+    CSipHasher& Write(uint64_t data);
+    /** Hash arbitrary bytes. */
+    CSipHasher& Write(const unsigned char* data, size_t size);
+    /** Compute the 64-bit SipHash-2-4 of the data written so far. The object remains untouched. */
+    uint64_t Finalize() const;
+};
+
+/** Optimized SipHash-2-4 implementation for uint256.
+ *
+ *  It is identical to:
+ *    SipHasher(k0, k1)
+ *      .Write(val.GetUint64(0))
+ *      .Write(val.GetUint64(1))
+ *      .Write(val.GetUint64(2))
+ *      .Write(val.GetUint64(3))
+ *      .Finalize()
+ */
+uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
+uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra);
+
+#endif // BITCOIN_CRYPTO_SIPHASH_H

src/hash.cpp

@@ -77,171 +77,3 @@ void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char he
     num[3] = (nChild >>  0) & 0xFF;
     CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output);
 }
-
-#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
-
-#define SIPROUND do { \
-    v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \
-    v0 = ROTL(v0, 32); \
-    v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \
-    v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \
-    v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \
-    v2 = ROTL(v2, 32); \
-} while (0)
-
-CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
-{
-    v[0] = 0x736f6d6570736575ULL ^ k0;
-    v[1] = 0x646f72616e646f6dULL ^ k1;
-    v[2] = 0x6c7967656e657261ULL ^ k0;
-    v[3] = 0x7465646279746573ULL ^ k1;
-    count = 0;
-    tmp = 0;
-}
-
-CSipHasher& CSipHasher::Write(uint64_t data)
-{
-    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
-
-    assert(count % 8 == 0);
-
-    v3 ^= data;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= data;
-
-    v[0] = v0;
-    v[1] = v1;
-    v[2] = v2;
-    v[3] = v3;
-
-    count += 8;
-    return *this;
-}
-
-CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size)
-{
-    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
-    uint64_t t = tmp;
-    int c = count;
-
-    while (size--) {
-        t |= ((uint64_t)(*(data++))) << (8 * (c % 8));
-        c++;
-        if ((c & 7) == 0) {
-            v3 ^= t;
-            SIPROUND;
-            SIPROUND;
-            v0 ^= t;
-            t = 0;
-        }
-    }
-
-    v[0] = v0;
-    v[1] = v1;
-    v[2] = v2;
-    v[3] = v3;
-    count = c;
-    tmp = t;
-
-    return *this;
-}
-
-uint64_t CSipHasher::Finalize() const
-{
-    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
-
-    uint64_t t = tmp | (((uint64_t)count) << 56);
-
-    v3 ^= t;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= t;
-    v2 ^= 0xFF;
-    SIPROUND;
-    SIPROUND;
-    SIPROUND;
-    SIPROUND;
-    return v0 ^ v1 ^ v2 ^ v3;
-}
-
-uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val)
-{
-    /* Specialized implementation for efficiency */
-    uint64_t d = val.GetUint64(0);
-
-    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
-    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
-    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
-    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
-
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    d = val.GetUint64(1);
-    v3 ^= d;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    d = val.GetUint64(2);
-    v3 ^= d;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    d = val.GetUint64(3);
-    v3 ^= d;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    v3 ^= ((uint64_t)4) << 59;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= ((uint64_t)4) << 59;
-    v2 ^= 0xFF;
-    SIPROUND;
-    SIPROUND;
-    SIPROUND;
-    SIPROUND;
-    return v0 ^ v1 ^ v2 ^ v3;
-}
-
-uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra)
-{
-    /* Specialized implementation for efficiency */
-    uint64_t d = val.GetUint64(0);
-
-    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
-    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
-    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
-    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
-
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    d = val.GetUint64(1);
-    v3 ^= d;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    d = val.GetUint64(2);
-    v3 ^= d;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    d = val.GetUint64(3);
-    v3 ^= d;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    d = (((uint64_t)36) << 56) | extra;
-    v3 ^= d;
-    SIPROUND;
-    SIPROUND;
-    v0 ^= d;
-    v2 ^= 0xFF;
-    SIPROUND;
-    SIPROUND;
-    SIPROUND;
-    SIPROUND;
-    return v0 ^ v1 ^ v2 ^ v3;
-}

src/hash.h

@@ -194,39 +194,4 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char
 
 void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]);
 
-/** SipHash-2-4 */
-class CSipHasher
-{
-private:
-    uint64_t v[4];
-    uint64_t tmp;
-    int count;
-
-public:
-    /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */
-    CSipHasher(uint64_t k0, uint64_t k1);
-    /** Hash a 64-bit integer worth of data
-     *  It is treated as if this was the little-endian interpretation of 8 bytes.
-     *  This function can only be used when a multiple of 8 bytes have been written so far.
-     */
-    CSipHasher& Write(uint64_t data);
-    /** Hash arbitrary bytes. */
-    CSipHasher& Write(const unsigned char* data, size_t size);
-    /** Compute the 64-bit SipHash-2-4 of the data written so far. The object remains untouched. */
-    uint64_t Finalize() const;
-};
-
-/** Optimized SipHash-2-4 implementation for uint256.
- *
- *  It is identical to:
- *    SipHasher(k0, k1)
- *      .Write(val.GetUint64(0))
- *      .Write(val.GetUint64(1))
- *      .Write(val.GetUint64(2))
- *      .Write(val.GetUint64(3))
- *      .Finalize()
- */
-uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
-uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra);
-
 #endif // BITCOIN_HASH_H

src/net.h

@@ -11,6 +11,7 @@
 #include <amount.h>
 #include <bloom.h>
 #include <compat.h>
+#include <crypto/siphash.h>
 #include <hash.h>
 #include <limitedmap.h>
 #include <netaddress.h>

src/test/hash_tests.cpp

@@ -2,6 +2,7 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <crypto/siphash.h>
 #include <hash.h>
 #include <util/strencodings.h>
 #include <test/test_bitcoin.h>

src/txmempool.h

@@ -15,6 +15,7 @@
 
 #include <amount.h>
 #include <coins.h>
+#include <crypto/siphash.h>
 #include <indirectmap.h>
 #include <policy/feerate.h>
 #include <primitives/transaction.h>

src/undo.h

@@ -11,6 +11,7 @@
 #include <consensus/consensus.h>
 #include <primitives/transaction.h>
 #include <serialize.h>
+#include <version.h>
 
 /** Undo information for a CTxIn
  *

src/util/bytevectorhash.cpp

@@ -0,0 +1,18 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <crypto/siphash.h>
+#include <random.h>
+#include <util/bytevectorhash.h>
+
+ByteVectorHash::ByteVectorHash()
+{
+    GetRandBytes(reinterpret_cast<unsigned char*>(&m_k0), sizeof(m_k0));
+    GetRandBytes(reinterpret_cast<unsigned char*>(&m_k1), sizeof(m_k1));
+}
+
+size_t ByteVectorHash::operator()(const std::vector<unsigned char>& input) const
+{
+    return CSipHasher(m_k0, m_k1).Write(input.data(), input.size()).Finalize();
+}

src/util/bytevectorhash.h

@@ -0,0 +1,26 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_UTIL_BYTEVECTORHASH_H
+#define BITCOIN_UTIL_BYTEVECTORHASH_H
+
+#include <stdint.h>
+#include <vector>
+
+/**
+ * Implementation of Hash named requirement for types that internally store a byte array. This may
+ * be used as the hash function in std::unordered_set or std::unordered_map over such types.
+ * Internally, this uses a random instance of SipHash-2-4.
+ */
+class ByteVectorHash final
+{
+private:
+    uint64_t m_k0, m_k1;
+
+public:
+    ByteVectorHash();
+    size_t operator()(const std::vector<unsigned char>& input) const;
+};
+
+#endif // BITCOIN_UTIL_BYTEVECTORHASH_H