From 4664f07b224d280756efcb981307a673055076e3 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Fri, 28 Feb 2020 22:11:08 -0700 Subject: [PATCH 1/4] make keystore and blockmap use hashtable --- .gitignore | 4 + src/claimtrie/blob.cpp | 12 ++- src/claimtrie/uints.h | 22 +++++ src/keystore.h | 9 +- src/pubkey.h | 11 +++ src/script/standard.h | 11 +++ src/validation.h | 18 ++-- src/wallet/crypter.h | 2 +- src/wallet/rpcwallet.cpp | 8 +- src/wallet/wallet.cpp | 184 ++++++++++++++++++++++----------------- src/wallet/wallet.h | 15 ++-- 11 files changed, 185 insertions(+), 111 deletions(-) diff --git a/.gitignore b/.gitignore index 932f17077..29e9b2228 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,7 @@ contrib/devtools/split-debug.sh cmake-build-*/ compile_commands\.json +**/compiler*\.d +conftest.dir/ + +confdefs.h diff --git a/src/claimtrie/blob.cpp b/src/claimtrie/blob.cpp index 58c91569b..a902b8dba 100644 --- a/src/claimtrie/blob.cpp +++ b/src/claimtrie/blob.cpp @@ -9,9 +9,8 @@ /** Template base class for fixed-sized opaque blobs. */ template -CBaseBlob::CBaseBlob() +CBaseBlob::CBaseBlob() : data{} { - SetNull(); } template @@ -30,8 +29,7 @@ CBaseBlob::CBaseBlob(const CBaseBlob& o) template CBaseBlob& CBaseBlob::operator=(const CBaseBlob& o) { - if (this != &o) - std::copy(o.begin(), o.end(), begin()); + data = o.data; return *this; } @@ -44,19 +42,19 @@ int CBaseBlob::Compare(const CBaseBlob& b) const template bool CBaseBlob::operator<(const CBaseBlob& b) const { - return Compare(b) < 0; + return data < b.data; } template bool CBaseBlob::operator==(const CBaseBlob& b) const { - return Compare(b) == 0; + return data == b.data; } template bool CBaseBlob::operator!=(const CBaseBlob& b) const { - return !(*this == b); + return data != b.data; } template diff --git a/src/claimtrie/uints.h b/src/claimtrie/uints.h index c6de8fd79..63efe0c53 100644 --- a/src/claimtrie/uints.h +++ b/src/claimtrie/uints.h @@ -40,4 +40,26 @@ uint160 uint160S(const std::string& s); uint256 uint256S(const char* str); uint256 uint256S(const std::string& s); +namespace std { + + template <> + struct hash + { + std::size_t operator()(const uint160& k) const + { + return *reinterpret_cast(k.begin()); + } + }; + + template <> + struct hash + { + std::size_t operator()(const uint256& k) const + { + return *reinterpret_cast(k.begin()); + } + }; +} + + #endif // CLAIMTRIE_UINTS_H diff --git a/src/keystore.h b/src/keystore.h index 4bd99e255..291ee0ef8 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -15,6 +15,9 @@ #include +#include +#include + /** A virtual base class for key stores */ class CKeyStore : public SigningProvider { @@ -44,9 +47,9 @@ class CBasicKeyStore : public CKeyStore protected: mutable CCriticalSection cs_KeyStore; - using KeyMap = std::map; - using WatchKeyMap = std::map; - using ScriptMap = std::map; + using KeyMap = std::unordered_map; + using WatchKeyMap = std::unordered_map; + using ScriptMap = std::unordered_map; using WatchOnlySet = std::set; KeyMap mapKeys GUARDED_BY(cs_KeyStore); diff --git a/src/pubkey.h b/src/pubkey.h index 2f36507fe..471c8dfc5 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -36,6 +36,17 @@ public: } }; +namespace std { + template <> + struct hash + { + std::size_t operator()(const CKeyID& k) const + { + return *reinterpret_cast(k.begin()); + } + }; +} + typedef uint256 ChainCode; /** An encapsulated public key. */ diff --git a/src/script/standard.h b/src/script/standard.h index b16ac83e7..d2ab00a11 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -27,6 +27,17 @@ public: CScriptID(const uint160& in) : uint160(in) {} }; +namespace std { + template <> + struct hash + { + std::size_t operator()(const CScriptID& k) const + { + return *reinterpret_cast(k.begin()); + } + }; +} + /** * Default setting for nMaxDatacarrierBytes. 80 bytes of data, +1 for OP_RETURN, * +2 for the pushdata opcodes. diff --git a/src/validation.h b/src/validation.h index 1acb6b95a..4083ced90 100644 --- a/src/validation.h +++ b/src/validation.h @@ -145,21 +145,27 @@ extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; extern std::atomic_bool g_is_mempool_loaded; -struct BlockIndexPointerCompare { - inline bool operator() (const CBlockIndex* lhs, const CBlockIndex* rhs) const { - return lhs->hash < rhs->hash; +struct BlockMapHasher { + std::size_t operator()(const CBlockIndex* block) const { + return *reinterpret_cast(block->hash.begin()); } }; -struct BlockMap : public std::set { +struct BlockMapComparer { + bool operator()(const CBlockIndex* a, const CBlockIndex* b) const { + return a == b || a->hash == b->hash; + } +}; + +struct BlockMap : public std::unordered_set { inline iterator find(const uint256& blockHash) { CBlockIndex temp(blockHash); - return std::set::find(&temp); + return std::unordered_set::find(&temp); } inline const_iterator find(const uint256& blockHash) const { CBlockIndex temp(blockHash); - return std::set::find(&temp); + return std::unordered_set::find(&temp); } }; diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 418316c39..6a7c76127 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -126,7 +126,7 @@ private: bool fDecryptionThoroughlyChecked; protected: - using CryptedKeyMap = std::map>>; + using CryptedKeyMap = std::unordered_map>>; bool SetCrypted(); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 8ab89857a..b770ff65b 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -467,15 +467,11 @@ static UniValue getaddressesbyaccount(const JSONRPCRequest& request) static CTransactionRef SendMoney(CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue, std::string fromAccount, const CScript& prefix = CScript()){ - CAmount curBalance = pwallet->GetBalance(); // Check amount if (nValue <= 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid amount"); - if (nValue > curBalance) - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds"); - if (pwallet->GetBroadcastTransactions() && !g_connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -493,6 +489,7 @@ static CTransactionRef SendMoney(CWallet * const pwallet, const CTxDestination & vecSend.push_back(recipient); CTransactionRef tx; if (!pwallet->CreateTransaction(vecSend, tx, reservekey, nFeeRequired, nChangePosRet, strError, coin_control)) { + CAmount curBalance = pwallet->GetBalance(); if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); throw JSONRPCError(RPC_WALLET_ERROR, strError); @@ -2193,7 +2190,6 @@ static UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bo for (auto item_it = start; item_it != end; ++item_it) { const CTxDestination& address = item_it->first; - const std::string& label = item_it->second.name; auto it = mapTally.find(address); if (it == mapTally.end() && !fIncludeEmpty) continue; @@ -2208,6 +2204,7 @@ static UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bo fIsWatchonly = (*it).second.fIsWatchonly; } + const std::string& label = item_it->second.name; if (by_label) { tallyitem& _item = label_tally[label]; @@ -2225,6 +2222,7 @@ static UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bo obj.pushKV("amount", ValueFromAmount(nAmount)); obj.pushKV("confirmations", (nConf == std::numeric_limits::max() ? 0 : nConf)); obj.pushKV("label", label); + obj.pushKV("purpose", item_it->second.purpose); UniValue transactions(UniValue::VARR); if (it != mapTally.end()) { diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e0dac9a6b..52c919c73 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -30,7 +30,7 @@ #include #include -#include +#include #include #include @@ -54,7 +54,7 @@ bool RemoveWallet(const std::shared_ptr& wallet) { LOCK(cs_wallets); assert(wallet); - std::vector>::iterator i = std::find(vpwallets.begin(), vpwallets.end(), wallet); + auto i = std::find(vpwallets.begin(), vpwallets.end(), wallet); if (i == vpwallets.end()) return false; vpwallets.erase(i); return true; @@ -208,7 +208,7 @@ public: const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const { LOCK(cs_wallet); - std::map::const_iterator it = mapWallet.find(hash); + auto it = mapWallet.find(hash); if (it == mapWallet.end()) return nullptr; return &(it->second); @@ -560,7 +560,7 @@ std::set CWallet::GetConflicts(const uint256& txid) const std::set result; AssertLockHeld(cs_wallet); - std::map::const_iterator it = mapWallet.find(txid); + auto it = mapWallet.find(txid); if (it == mapWallet.end()) return result; const CWalletTx& wtx = it->second; @@ -569,11 +569,16 @@ std::set CWallet::GetConflicts(const uint256& txid) const for (const CTxIn& txin : wtx.tx->vin) { - if (mapTxSpends.count(txin.prevout) <= 1) + auto hitTx = mapTxSpends.find(txin.prevout.hash); + if (hitTx == mapTxSpends.end()) + continue; + auto hitN = hitTx->second.find(txin.prevout.n); + if (hitN == hitTx->second.end()) + continue; + if (hitN->second.size() <= 1U) continue; // No conflict if zero or one spends - range = mapTxSpends.equal_range(txin.prevout); - for (TxSpends::const_iterator _it = range.first; _it != range.second; ++_it) - result.insert(_it->second); + for (auto& spend: hitN->second) + result.insert(spend); } return result; } @@ -581,8 +586,14 @@ std::set CWallet::GetConflicts(const uint256& txid) const bool CWallet::HasWalletSpend(const uint256& txid) const { AssertLockHeld(cs_wallet); - auto iter = mapTxSpends.lower_bound(COutPoint(txid, 0)); - return (iter != mapTxSpends.end() && iter->first.hash == txid); + auto hitTx = mapTxSpends.find(txid); + if (hitTx == mapTxSpends.end()) + return false; + for (auto& kvp: hitTx->second) { + if (!kvp.second.empty()) + return true; + } + return false; } void CWallet::Flush(bool shutdown) @@ -590,7 +601,7 @@ void CWallet::Flush(bool shutdown) database->Flush(shutdown); } -void CWallet::SyncMetaData(std::pair range) +void CWallet::SyncMetaData(const COutPoint& outPoint) { // We want all the wallet transactions in range to have the same metadata as // the oldest (smallest nOrderPos). @@ -598,36 +609,40 @@ void CWallet::SyncMetaData(std::pair ran int nMinOrderPos = std::numeric_limits::max(); const CWalletTx* copyFrom = nullptr; - for (TxSpends::iterator it = range.first; it != range.second; ++it) { - const CWalletTx* wtx = &mapWallet.at(it->second); - if (wtx->nOrderPos < nMinOrderPos) { - nMinOrderPos = wtx->nOrderPos; - copyFrom = wtx; + + auto hitTx = mapTxSpends.find(outPoint.hash); + if (hitTx != mapTxSpends.end()) { + auto hitN = hitTx->second.find(outPoint.n); + if (hitN != hitTx->second.end()) { + for (auto &spend: hitN->second) { + const CWalletTx *wtx = &mapWallet.at(spend); + if (wtx->nOrderPos < nMinOrderPos) { + nMinOrderPos = wtx->nOrderPos; + copyFrom = wtx; + } + } + if (!copyFrom) { + return; + } + + // Now copy data from copyFrom to rest: + for (auto &hash: hitN->second) { + CWalletTx *copyTo = &mapWallet.at(hash); + if (copyFrom == copyTo) continue; + assert(copyFrom && "Oldest wallet transaction in range assumed to have been found."); + if (!copyFrom->IsEquivalentTo(*copyTo)) continue; + copyTo->mapValue = copyFrom->mapValue; + copyTo->vOrderForm = copyFrom->vOrderForm; + // fTimeReceivedIsTxTime not copied on purpose + // nTimeReceived not copied on purpose + copyTo->nTimeSmart = copyFrom->nTimeSmart; + copyTo->fFromMe = copyFrom->fFromMe; + copyTo->strFromAccount = copyFrom->strFromAccount; + // nOrderPos not copied on purpose + // cached members not copied on purpose + } } } - - if (!copyFrom) { - return; - } - - // Now copy data from copyFrom to rest: - for (TxSpends::iterator it = range.first; it != range.second; ++it) - { - const uint256& hash = it->second; - CWalletTx* copyTo = &mapWallet.at(hash); - if (copyFrom == copyTo) continue; - assert(copyFrom && "Oldest wallet transaction in range assumed to have been found."); - if (!copyFrom->IsEquivalentTo(*copyTo)) continue; - copyTo->mapValue = copyFrom->mapValue; - copyTo->vOrderForm = copyFrom->vOrderForm; - // fTimeReceivedIsTxTime not copied on purpose - // nTimeReceived not copied on purpose - copyTo->nTimeSmart = copyFrom->nTimeSmart; - copyTo->fFromMe = copyFrom->fFromMe; - copyTo->strFromAccount = copyFrom->strFromAccount; - // nOrderPos not copied on purpose - // cached members not copied on purpose - } } /** @@ -636,14 +651,15 @@ void CWallet::SyncMetaData(std::pair ran */ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const { - const COutPoint outpoint(hash, n); - std::pair range; - range = mapTxSpends.equal_range(outpoint); + auto hitTx = mapTxSpends.find(hash); + if (hitTx == mapTxSpends.end()) + return false; + auto hitN = hitTx->second.find(n); + if (hitN == hitTx->second.end()) + return false; - for (TxSpends::const_iterator it = range.first; it != range.second; ++it) - { - const uint256& wtxid = it->second; - std::map::const_iterator mit = mapWallet.find(wtxid); + for (auto& wtxid: hitN->second) { + auto mit = mapWallet.find(wtxid); if (mit != mapWallet.end()) { int depth = mit->second.GetDepthInMainChain(); if (depth > 0 || (depth == 0 && !mit->second.isAbandoned())) @@ -655,13 +671,9 @@ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid) { - mapTxSpends.insert(std::make_pair(outpoint, wtxid)); - + mapTxSpends[outpoint.hash][outpoint.n].push_back(wtxid); setLockedCoins.erase(outpoint); - - std::pair range; - range = mapTxSpends.equal_range(outpoint); - SyncMetaData(range); + SyncMetaData(outpoint); } @@ -907,9 +919,7 @@ bool CWallet::GetLabelDestination(CTxDestination &dest, const std::string& label else { // Check if the current key has been used (TODO: check other addresses with the same key) CScript scriptPubKey = GetScriptForDestination(GetDestinationForKey(account.vchPubKey, m_default_address_type)); - for (std::map::iterator it = mapWallet.begin(); - it != mapWallet.end() && account.vchPubKey.IsValid(); - ++it) + for (auto it = mapWallet.begin(); it != mapWallet.end() && account.vchPubKey.IsValid(); ++it) for (const CTxOut& txout : (*it).second.tx->vout) if (txout.scriptPubKey == scriptPubKey) { bForceNew = true; @@ -981,7 +991,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) uint256 hash = wtxIn.GetHash(); // Inserts only if not already there, returns tx inserted or tx found - std::pair::iterator, bool> ret = mapWallet.insert(std::make_pair(hash, wtxIn)); + auto ret = mapWallet.insert(std::make_pair(hash, wtxIn)); CWalletTx& wtx = (*ret.first).second; wtx.BindWallet(this); bool fInsertedNew = ret.second; @@ -1086,13 +1096,19 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI if (pIndex != nullptr) { for (const CTxIn& txin : tx.vin) { - std::pair range = mapTxSpends.equal_range(txin.prevout); - while (range.first != range.second) { - if (range.first->second != tx.GetHash()) { - WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), pIndex->GetBlockHash().ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); - MarkConflicted(pIndex->GetBlockHash(), range.first->second); + auto hitTx = mapTxSpends.find(txin.prevout.hash); + if (hitTx == mapTxSpends.end()) + continue; + auto hitN = hitTx->second.find(txin.prevout.n); + if (hitN == hitTx->second.end()) + continue; + + for (auto& spend: hitN->second) { + if (spend != tx.GetHash()) { + WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", + tx.GetHash().ToString(), pIndex->GetBlockHash().ToString(), spend.ToString(), hitTx->first.ToString(), hitN->first); + MarkConflicted(pIndex->GetBlockHash(), spend); } - range.first++; } } } @@ -1113,7 +1129,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI std::vector vAffected; CAffectedKeysVisitor(*this, vAffected).Process(txout.scriptPubKey); for (const CKeyID &keyid : vAffected) { - std::map::const_iterator mi = m_pool_key_to_index.find(keyid); + auto mi = m_pool_key_to_index.find(keyid); if (mi != m_pool_key_to_index.end()) { WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); MarkReserveKeysAsUsed(mi->second); @@ -1177,7 +1193,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) uint256 now = *todo.begin(); todo.erase(now); done.insert(now); - auto it = mapWallet.find(now); + it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; int currentconfirm = wtx.GetDepthInMainChain(); @@ -1193,12 +1209,14 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) batch.WriteTx(wtx); NotifyTransactionChanged(this, wtx.GetHash(), CT_UPDATED); // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too - TxSpends::const_iterator iter = mapTxSpends.lower_bound(COutPoint(now, 0)); - while (iter != mapTxSpends.end() && iter->first.hash == now) { - if (!done.count(iter->second)) { - todo.insert(iter->second); + auto hitTx = mapTxSpends.find(now); + if (hitTx != mapTxSpends.end()) { + for (auto &kvp: hitTx->second) { + for (auto &spent: kvp.second) { + if (!done.count(spent)) + todo.insert(spent); + } } - iter++; } // If a transaction changes 'conflicted' state, that changes the balance // available of the outputs it spends. So force those to be recomputed @@ -1249,12 +1267,14 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) wtx.MarkDirty(); batch.WriteTx(wtx); // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too - TxSpends::const_iterator iter = mapTxSpends.lower_bound(COutPoint(now, 0)); - while (iter != mapTxSpends.end() && iter->first.hash == now) { - if (!done.count(iter->second)) { - todo.insert(iter->second); - } - iter++; + auto hitTx = mapTxSpends.find(now); + if (hitTx != mapTxSpends.end()) { + for (auto &kvp: hitTx->second) { + for (auto &spent: kvp.second) { + if (!done.count(spent)) + todo.insert(spent); + } + } } // If a transaction changes 'conflicted' state, that changes the balance // available of the outputs it spends. So force those to be recomputed @@ -1352,7 +1372,7 @@ isminetype CWallet::IsMine(const CTxIn &txin) const { { LOCK(cs_wallet); - std::map::const_iterator mi = mapWallet.find(txin.prevout.hash); + auto mi = mapWallet.find(txin.prevout.hash); if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; @@ -1369,7 +1389,7 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const { { LOCK(cs_wallet); - std::map::const_iterator mi = mapWallet.find(txin.prevout.hash); + auto mi = mapWallet.find(txin.prevout.hash); if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; @@ -2388,7 +2408,7 @@ void CWallet::AvailableCoins(std::vector &vCoins, bool fOnlySafe, const solvable = IsSolvable(*this, pcoin->tx->vout[i].scriptPubKey); // this is a slow call spendable = solvable; computedSolvable = true; - }; + } if (computeSolvable && !computedSolvable) solvable = IsSolvable(*this, pcoin->tx->vout[i].scriptPubKey); @@ -2560,7 +2580,7 @@ bool CWallet::SelectCoins(const std::vector& vAvailableCoins, const CAm bnb_used = false; coin_selection_params.use_bnb = false; - std::map::const_iterator it = mapWallet.find(outpoint.hash); + auto it = mapWallet.find(outpoint.hash); if (it != mapWallet.end()) { const CWalletTx* pcoin = &it->second; @@ -2622,7 +2642,7 @@ bool CWallet::SignTransaction(CMutableTransaction &tx) // sign the new tx int nIn = 0; for (auto& input : tx.vin) { - std::map::const_iterator mi = mapWallet.find(input.prevout.hash); + auto mi = mapWallet.find(input.prevout.hash); if(mi == mapWallet.end() || input.prevout.n >= mi->second.tx->vout.size()) { return false; } @@ -3886,7 +3906,7 @@ void CWallet::GetKeyBirthTimes(std::map &mapKeyBirth) c CAffectedKeysVisitor(*this, vAffected).Process(txout.scriptPubKey); for (const CKeyID &keyid : vAffected) { // ... and all their affected keys - std::map::iterator rit = mapKeyFirstBlock.find(keyid); + auto rit = mapKeyFirstBlock.find(keyid); if (rit != mapKeyFirstBlock.end() && nHeight < rit->second->nHeight) rit->second = pindex; } @@ -4381,7 +4401,7 @@ std::shared_ptr CWallet::CreateWalletFromFile(const WalletLocation& loc for (const CWalletTx& wtxOld : vWtx) { uint256 hash = wtxOld.GetHash(); - std::map::iterator mi = walletInstance->mapWallet.find(hash); + auto mi = walletInstance->mapWallet.find(hash); if (mi != walletInstance->mapWallet.end()) { const CWalletTx* copyFrom = &wtxOld; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6abef757a..43000d768 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -28,8 +28,8 @@ #include #include #include -#include #include +#include #include #include @@ -699,7 +699,7 @@ private: * detect and report conflicts (double-spends or * mutated transactions where the mutant gets mined). */ - typedef std::multimap TxSpends; + typedef std::unordered_map>> TxSpends; TxSpends mapTxSpends; void AddToSpends(const COutPoint& outpoint, const uint256& wtxid); void AddToSpends(const uint256& wtxid); @@ -725,7 +725,7 @@ private: /* Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx); - void SyncMetaData(std::pair); + void SyncMetaData(const COutPoint& outpoint); /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected/ScanForWalletTransactions. * Should be called with pindexBlock and posInBlock if this is for a transaction that is included in a block. */ @@ -808,17 +808,18 @@ public: void MarkPreSplitKeys(); // Map from Key ID to key metadata. - std::map mapKeyMetadata; + std::unordered_map mapKeyMetadata; // Map from Script ID to key metadata (for watch-only keys). - std::map m_script_metadata; + std::unordered_map m_script_metadata; typedef std::map MasterKeyMap; MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID = 0; /** Construct wallet with specified name and database implementation. */ - CWallet(const WalletLocation& location, std::unique_ptr database) : m_location(location), database(std::move(database)) + CWallet(const WalletLocation& location, std::unique_ptr database) + : m_location(location), database(std::move(database)) { } @@ -830,7 +831,7 @@ public: encrypted_batch = nullptr; } - std::map mapWallet; + std::unordered_map mapWallet; std::list laccentries; typedef std::pair TxPair; From b2186806e3699efaa0ebb37c602629256f986cc5 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Tue, 3 Mar 2020 09:03:51 -0700 Subject: [PATCH 2/4] rob the rich to pay the poor --- src/Makefile.am | 1 + src/claimtrie/blob.cpp | 6 +- src/claimtrie/blob.h | 6 +- src/claimtrie/uints.h | 8 +- src/keystore.h | 8 +- src/primitives/robin_hood.h | 2198 +++++++++++++++++++++++++++++++++++ src/pubkey.h | 4 +- src/script/standard.h | 4 +- src/validation.h | 7 +- src/wallet/rpcwallet.cpp | 10 +- src/wallet/wallet.cpp | 8 +- src/wallet/wallet.h | 10 +- 12 files changed, 2235 insertions(+), 35 deletions(-) create mode 100644 src/primitives/robin_hood.h diff --git a/src/Makefile.am b/src/Makefile.am index 447e48f52..859c5fccc 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -148,6 +148,7 @@ BITCOIN_CORE_H = \ policy/fees.h \ policy/policy.h \ policy/rbf.h \ + primitives/robin_hood.h \ pow.h \ protocol.h \ random.h \ diff --git a/src/claimtrie/blob.cpp b/src/claimtrie/blob.cpp index a902b8dba..6f30f440b 100644 --- a/src/claimtrie/blob.cpp +++ b/src/claimtrie/blob.cpp @@ -9,7 +9,7 @@ /** Template base class for fixed-sized opaque blobs. */ template -CBaseBlob::CBaseBlob() : data{} +CBaseBlob::CBaseBlob() noexcept : data{} { } @@ -21,13 +21,13 @@ CBaseBlob::CBaseBlob(const std::vector& vec) } template -CBaseBlob::CBaseBlob(const CBaseBlob& o) +CBaseBlob::CBaseBlob(const CBaseBlob& o) noexcept { *this = o; } template -CBaseBlob& CBaseBlob::operator=(const CBaseBlob& o) +CBaseBlob& CBaseBlob::operator=(const CBaseBlob& o) noexcept { data = o.data; return *this; diff --git a/src/claimtrie/blob.h b/src/claimtrie/blob.h index 20df687d9..c42034f31 100644 --- a/src/claimtrie/blob.h +++ b/src/claimtrie/blob.h @@ -12,15 +12,15 @@ class CBaseBlob { std::array data; public: - CBaseBlob(); + CBaseBlob() noexcept; explicit CBaseBlob(const std::vector& vec); CBaseBlob(CBaseBlob&&) = default; CBaseBlob& operator=(CBaseBlob&&) = default; - CBaseBlob(const CBaseBlob& o); - CBaseBlob& operator=(const CBaseBlob& o); + CBaseBlob(const CBaseBlob& o) noexcept; + CBaseBlob& operator=(const CBaseBlob& o) noexcept; int Compare(const CBaseBlob& b) const; bool operator<(const CBaseBlob& b) const; diff --git a/src/claimtrie/uints.h b/src/claimtrie/uints.h index 63efe0c53..1420d8ec9 100644 --- a/src/claimtrie/uints.h +++ b/src/claimtrie/uints.h @@ -45,18 +45,18 @@ namespace std { template <> struct hash { - std::size_t operator()(const uint160& k) const + size_t operator()(const uint160& k) const { - return *reinterpret_cast(k.begin()); + return *reinterpret_cast(k.begin()); } }; template <> struct hash { - std::size_t operator()(const uint256& k) const + size_t operator()(const uint256& k) const { - return *reinterpret_cast(k.begin()); + return *reinterpret_cast(k.begin()); } }; } diff --git a/src/keystore.h b/src/keystore.h index 291ee0ef8..c67b7d567 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -15,7 +15,7 @@ #include -#include +#include #include /** A virtual base class for key stores */ @@ -47,9 +47,9 @@ class CBasicKeyStore : public CKeyStore protected: mutable CCriticalSection cs_KeyStore; - using KeyMap = std::unordered_map; - using WatchKeyMap = std::unordered_map; - using ScriptMap = std::unordered_map; + using KeyMap = robin_hood::unordered_map; + using WatchKeyMap = robin_hood::unordered_map; + using ScriptMap = robin_hood::unordered_map; using WatchOnlySet = std::set; KeyMap mapKeys GUARDED_BY(cs_KeyStore); diff --git a/src/primitives/robin_hood.h b/src/primitives/robin_hood.h new file mode 100644 index 000000000..2fa21f27e --- /dev/null +++ b/src/primitives/robin_hood.h @@ -0,0 +1,2198 @@ +// ______ _____ ______ _________ +// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ / +// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ / +// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ / +// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/ +// _/_____/ +// +// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20 +// version 3.5.2 +// https://github.com/martinus/robin-hood-hashing +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2020 Martin Ankerl +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ROBIN_HOOD_H_INCLUDED +#define ROBIN_HOOD_H_INCLUDED + +// see https://semver.org/ +#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes +#define ROBIN_HOOD_VERSION_MINOR 5 // for adding functionality in a backwards-compatible manner +#define ROBIN_HOOD_VERSION_PATCH 2 // for backwards-compatible bug fixes + +#include +#include +#include +#include +#include +#include +#include +#include + +// #define ROBIN_HOOD_LOG_ENABLED +#ifdef ROBIN_HOOD_LOG_ENABLED +# include +# define ROBIN_HOOD_LOG(x) std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << x << std::endl +#else +# define ROBIN_HOOD_LOG(x) +#endif + +// #define ROBIN_HOOD_TRACE_ENABLED +#ifdef ROBIN_HOOD_TRACE_ENABLED +# include +# define ROBIN_HOOD_TRACE(x) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << x << std::endl +#else +# define ROBIN_HOOD_TRACE(x) +#endif + +// #define ROBIN_HOOD_COUNT_ENABLED +#ifdef ROBIN_HOOD_COUNT_ENABLED +# include +# define ROBIN_HOOD_COUNT(x) ++counts().x; +namespace robin_hood { +struct Counts { + uint64_t shiftUp{}; + uint64_t shiftDown{}; +}; +inline std::ostream& operator<<(std::ostream& os, Counts const& c) { + return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl; +} + +static Counts& counts() { + static Counts counts{}; + return counts; +} +} // namespace robin_hood +#else +# define ROBIN_HOOD_COUNT(x) +#endif + +// all non-argument macros should use this facility. See +// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/ +#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x() + +// mark unused members with this macro +#define ROBIN_HOOD_UNUSED(identifier) + +// bitness +#if SIZE_MAX == UINT32_MAX +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32 +#elif SIZE_MAX == UINT64_MAX +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64 +#else +# error Unsupported bitness +#endif + +// endianess +#ifdef _MSC_VER +# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0 +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \ + (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#endif + +// inline +#ifdef _MSC_VER +# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline) +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline)) +#endif + +// exceptions +#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0 +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1 +#endif + +// count leading/trailing bits +#ifdef _MSC_VER +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 +# endif +# include +# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ + [](size_t mask) noexcept -> int { \ + unsigned long index; \ + return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast(index) \ + : ROBIN_HOOD(BITNESS); \ + }(x) +#else +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll +# endif +# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) +#endif + +// fallthrough +#ifndef __has_cpp_attribute // For backwards compatibility +# define __has_cpp_attribute(x) 0 +#endif +#if __has_cpp_attribute(clang::fallthrough) +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]] +#elif __has_cpp_attribute(gnu::fallthrough) +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]] +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() +#endif + +// likely/unlikely +#ifdef _MSC_VER +# define ROBIN_HOOD_LIKELY(condition) condition +# define ROBIN_HOOD_UNLIKELY(condition) condition +#else +# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1) +# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0) +#endif + +// workaround missing "is_trivially_copyable" in g++ < 5.0 +// See https://stackoverflow.com/a/31798726/48181 +#if defined(__GNUC__) && __GNUC__ < 5 +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) +#else +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value +#endif + +// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) +# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]] +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() +#endif + +namespace robin_hood { + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) +# define ROBIN_HOOD_STD std +#else + + // c++11 compatibility layer +namespace ROBIN_HOOD_STD { +template +struct alignment_of + : std::integral_constant::type)> {}; + +template +class integer_sequence { +public: + using value_type = T; + static_assert(std::is_integral::value, "not integral type"); + static constexpr std::size_t size() noexcept { + return sizeof...(Ints); + } +}; +template +using index_sequence = integer_sequence; + +namespace detail_ { +template +struct IntSeqImpl { + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)"); + + template + struct IntSeqCombiner; + + template + struct IntSeqCombiner, integer_sequence> { + using TResult = integer_sequence; + }; + + using TResult = + typename IntSeqCombiner::TResult, + typename IntSeqImpl::TResult>::TResult; +}; + +template +struct IntSeqImpl { + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence; +}; + +template +struct IntSeqImpl { + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence; +}; +} // namespace detail_ + +template +using make_integer_sequence = typename detail_::IntSeqImpl::TResult; + +template +using make_index_sequence = make_integer_sequence; + +template +using index_sequence_for = make_index_sequence; + +} // namespace ROBIN_HOOD_STD + +#endif + + namespace detail { + +// umul +#if defined(__SIZEOF_INT128__) +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 1 +# if defined(__GNUC__) || defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpedantic" + using uint128_t = unsigned __int128; +# pragma GCC diagnostic pop +# endif + inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) noexcept { + auto result = static_cast(a) * static_cast(b); + *high = static_cast(result >> 64U); + return static_cast(result); + } +#elif (defined(_MSC_VER) && ROBIN_HOOD(BITNESS) == 64) + # define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 1 +# include // for __umulh +# pragma intrinsic(__umulh) +# ifndef _M_ARM64 +# pragma intrinsic(_umul128) +# endif +inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) noexcept { +# ifdef _M_ARM64 + *high = __umulh(a, b); + return ((uint64_t)(a)) * (b); +# else + return _umul128(a, b, high); +# endif +} +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 0 +#endif + +// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to +// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with +// care! + template + inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept { + return reinterpret_cast(ptr); + } + + template + inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept { + return reinterpret_cast(ptr); + } + +// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other +// inlinings more difficult. Throws are also generally the slow path. + template + ROBIN_HOOD(NOINLINE) +#if ROBIN_HOOD(HAS_EXCEPTIONS) + void doThrow(Args&&... args) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) + throw E(std::forward(args)...); + } +#else + void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) { + abort(); +} +#endif + + template + T* assertNotNull(T* t, Args&&... args) { + if (ROBIN_HOOD_UNLIKELY(nullptr == t)) { + doThrow(std::forward(args)...); + } + return t; + } + + template + inline T unaligned_load(void const* ptr) noexcept { + // using memcpy so we don't get into unaligned load problems. + // compiler should optimize this very well anyways. + T t; + std::memcpy(&t, ptr, sizeof(T)); + return t; + } + +// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor, +// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a +// pointer. + template + class BulkPoolAllocator { + public: + BulkPoolAllocator() noexcept = default; + + // does not copy anything, just creates a new allocator. + BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept + : mHead(nullptr) + , mListForFree(nullptr) {} + + BulkPoolAllocator(BulkPoolAllocator&& o) noexcept + : mHead(o.mHead) + , mListForFree(o.mListForFree) { + o.mListForFree = nullptr; + o.mHead = nullptr; + } + + BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept { + reset(); + mHead = o.mHead; + mListForFree = o.mListForFree; + o.mListForFree = nullptr; + o.mHead = nullptr; + return *this; + } + + BulkPoolAllocator& + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept { + // does not do anything + return *this; + } + + ~BulkPoolAllocator() noexcept { + reset(); + } + + // Deallocates all allocated memory. + void reset() noexcept { + while (mListForFree) { + T* tmp = *mListForFree; + free(mListForFree); + mListForFree = reinterpret_cast_no_cast_align_warning(tmp); + } + mHead = nullptr; + } + + // allocates, but does NOT initialize. Use in-place new constructor, e.g. + // T* obj = pool.allocate(); + // ::new (static_cast(obj)) T(); + T* allocate() { + T* tmp = mHead; + if (!tmp) { + tmp = performAllocation(); + } + + mHead = *reinterpret_cast_no_cast_align_warning(tmp); + return tmp; + } + + // does not actually deallocate but puts it in store. + // make sure you have already called the destructor! e.g. with + // obj->~T(); + // pool.deallocate(obj); + void deallocate(T* obj) noexcept { + *reinterpret_cast_no_cast_align_warning(obj) = mHead; + mHead = obj; + } + + // Adds an already allocated block of memory to the allocator. This allocator is from now on + // responsible for freeing the data (with free()). If the provided data is not large enough to + // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor. + void addOrFree(void* ptr, const size_t numBytes) noexcept { + // calculate number of available elements in ptr + if (numBytes < ALIGNMENT + ALIGNED_SIZE) { + // not enough data for at least one element. Free and return. + free(ptr); + } else { + add(ptr, numBytes); + } + } + + void swap(BulkPoolAllocator& other) noexcept { + using std::swap; + swap(mHead, other.mHead); + swap(mListForFree, other.mListForFree); + } + + private: + // iterates the list of allocated memory to calculate how many to alloc next. + // Recalculating this each time saves us a size_t member. + // This ignores the fact that memory blocks might have been added manually with addOrFree. In + // practice, this should not matter much. + ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept { + auto tmp = mListForFree; + size_t numAllocs = MinNumAllocs; + + while (numAllocs * 2 <= MaxNumAllocs && tmp) { + auto x = reinterpret_cast(tmp); + tmp = *x; + numAllocs *= 2; + } + + return numAllocs; + } + + // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree(). + void add(void* ptr, const size_t numBytes) noexcept { + const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE; + + auto data = reinterpret_cast(ptr); + + // link free list + auto x = reinterpret_cast(data); + *x = mListForFree; + mListForFree = data; + + // create linked list for newly allocated data + auto const headT = + reinterpret_cast_no_cast_align_warning(reinterpret_cast(ptr) + ALIGNMENT); + + auto const head = reinterpret_cast(headT); + + // Visual Studio compiler automatically unrolls this loop, which is pretty cool + for (size_t i = 0; i < numElements; ++i) { + *reinterpret_cast_no_cast_align_warning(head + i * ALIGNED_SIZE) = + head + (i + 1) * ALIGNED_SIZE; + } + + // last one points to 0 + *reinterpret_cast_no_cast_align_warning(head + (numElements - 1) * ALIGNED_SIZE) = + mHead; + mHead = headT; + } + + // Called when no memory is available (mHead == 0). + // Don't inline this slow path. + ROBIN_HOOD(NOINLINE) T* performAllocation() { + size_t const numElementsToAlloc = calcNumElementsToAlloc(); + + // alloc new memory: [prev |T, T, ... T] + // std::cout << (sizeof(T*) + ALIGNED_SIZE * numElementsToAlloc) << " bytes" << std::endl; + size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc; + add(assertNotNull(malloc(bytes)), bytes); + return mHead; + } + + // enforce byte alignment of the T's +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) + static constexpr size_t ALIGNMENT = + (std::max)(std::alignment_of::value, std::alignment_of::value); +#else + static const size_t ALIGNMENT = + (ROBIN_HOOD_STD::alignment_of::value > ROBIN_HOOD_STD::alignment_of::value) + ? ROBIN_HOOD_STD::alignment_of::value + : +ROBIN_HOOD_STD::alignment_of::value; // the + is for walkarround +#endif + + static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT; + + static_assert(MinNumAllocs >= 1, "MinNumAllocs"); + static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs"); + static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE"); + static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod"); + static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT"); + + T* mHead{nullptr}; + T** mListForFree{nullptr}; + }; + + template + struct NodeAllocator; + +// dummy allocator that does nothing + template + struct NodeAllocator { + + // we are not using the data, so just free it. + void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept { + free(ptr); + } + }; + + template + struct NodeAllocator : public BulkPoolAllocator {}; + +// dummy hash, unsed as mixer when robin_hood::hash is already used + template + struct identity_hash { + constexpr size_t operator()(T const& obj) const noexcept { + return static_cast(obj); + } + }; + +// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making +// my own here. + namespace swappable { + using std::swap; + template + struct nothrow { + static const bool value = noexcept(swap(std::declval(), std::declval())); + }; + + } // namespace swappable + + } // namespace detail + + struct is_transparent_tag {}; + +// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable, +// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is +// also tested. + template + struct pair { + using first_type = T1; + using second_type = T2; + + template ::value && + std::is_default_constructible::value>::type> + constexpr pair() noexcept(noexcept(U1()) && noexcept(U2())) + : first() + , second() {} + + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair const& o) noexcept( + noexcept(T1(std::declval())) && noexcept(T2(std::declval()))) + : first(o.first) + , second(o.second) {} + + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair&& o) noexcept( + noexcept(T1(std::move(std::declval()))) && + noexcept(T2(std::move(std::declval())))) + : first(std::move(o.first)) + , second(std::move(o.second)) {} + + constexpr pair(T1&& a, T2&& b) noexcept(noexcept(T1(std::move(std::declval()))) && + noexcept(T2(std::move(std::declval())))) + : first(std::move(a)) + , second(std::move(b)) {} + + template + constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward(std::declval()))) && + noexcept(T2(std::forward(std::declval())))) + : first(std::forward(a)) + , second(std::forward(b)) {} + + template + constexpr pair( + std::piecewise_construct_t /*unused*/, std::tuple a, + std::tuple b) noexcept(noexcept(pair(std::declval&>(), + std::declval&>(), + ROBIN_HOOD_STD::index_sequence_for(), + ROBIN_HOOD_STD::index_sequence_for()))) + : pair(a, b, ROBIN_HOOD_STD::index_sequence_for(), + ROBIN_HOOD_STD::index_sequence_for()) {} + + // constructor called from the std::piecewise_construct_t ctor + template + pair(std::tuple& a, std::tuple& b, + ROBIN_HOOD_STD::index_sequence /*unused*/, + ROBIN_HOOD_STD::index_sequence< + I2...> /*unused*/) noexcept(noexcept(T1(std:: + forward(std::get( + std::declval< + std::tuple&>()))...)) && + noexcept(T2(std::forward( + std::get(std::declval&>()))...))) + : first(std::forward(std::get(a))...) + , second(std::forward(std::get(b))...) { + // make visual studio compiler happy about warning about unused a & b. + // Visual studio's pair implementation disables warning 4100. + (void)a; + (void)b; + } + + void swap(pair& o) noexcept((detail::swappable::nothrow::value) && + (detail::swappable::nothrow::value)) { + using std::swap; + swap(first, o.first); + swap(second, o.second); + } + + T1 first; // NOLINT(misc-non-private-member-variables-in-classes) + T2 second; // NOLINT(misc-non-private-member-variables-in-classes) + }; + + template + void swap(pair& a, pair& b) noexcept( + noexcept(std::declval&>().swap(std::declval&>()))) { + a.swap(b); + } + +// Hash an arbitrary amount of bytes. This is basically Murmur2 hash without caring about big +// endianness. TODO(martinus) add a fallback for very large strings? + static size_t hash_bytes(void const* ptr, size_t const len) noexcept { + static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); + static constexpr uint64_t seed = UINT64_C(0xe17a1465); + static constexpr unsigned int r = 47; + + auto const data64 = static_cast(ptr); + uint64_t h = seed ^ (len * m); + + size_t const n_blocks = len / 8; + for (size_t i = 0; i < n_blocks; ++i) { + auto k = detail::unaligned_load(data64 + i); + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + auto const data8 = reinterpret_cast(data64 + n_blocks); + switch (len & 7U) { + case 7: + h ^= static_cast(data8[6]) << 48U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 6: + h ^= static_cast(data8[5]) << 40U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 5: + h ^= static_cast(data8[4]) << 32U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 4: + h ^= static_cast(data8[3]) << 24U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 3: + h ^= static_cast(data8[2]) << 16U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 2: + h ^= static_cast(data8[1]) << 8U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 1: + h ^= static_cast(data8[0]); + h *= m; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + default: + break; + } + + h ^= h >> r; + h *= m; + h ^= h >> r; + return static_cast(h); + } + + inline size_t hash_int(uint64_t obj) noexcept { +#if ROBIN_HOOD(HAS_UMUL128) + // 167079903232 masksum, 120428523 ops best: 0xde5fb9d2630458e9 + static constexpr uint64_t k = UINT64_C(0xde5fb9d2630458e9); + uint64_t h; + uint64_t l = detail::umul128(obj, k, &h); + return h + l; +#elif ROBIN_HOOD(BITNESS) == 32 + uint64_t const r = obj * UINT64_C(0xca4bcaa75ec3f625); + auto h = static_cast(r >> 32U); + auto l = static_cast(r); + return h + l; +#else + // murmurhash 3 finalizer + uint64_t h = obj; + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + h ^= h >> 33; + return static_cast(h); +#endif + } + +// A thin wrapper around std::hash, performing an additional simple mixing step of the result. + template + struct hash : public std::hash { + size_t operator()(T const& obj) const + noexcept(noexcept(std::declval>().operator()(std::declval()))) { + // call base hash + auto result = std::hash::operator()(obj); + // return mixed of that, to be save against identity has + return hash_int(static_cast(result)); + } + }; + + template <> + struct hash { + size_t operator()(std::string const& str) const noexcept { + return hash_bytes(str.data(), str.size()); + } + }; + + template + struct hash { + size_t operator()(T* ptr) const noexcept { + return hash_int(reinterpret_cast(ptr)); + } + }; + +#define ROBIN_HOOD_HASH_INT(T) \ + template <> \ + struct hash { \ + size_t operator()(T obj) const noexcept { \ + return hash_int(static_cast(obj)); \ + } \ + } + +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wuseless-cast" +#endif +// see https://en.cppreference.com/w/cpp/utility/hash + ROBIN_HOOD_HASH_INT(bool); + ROBIN_HOOD_HASH_INT(char); + ROBIN_HOOD_HASH_INT(signed char); + ROBIN_HOOD_HASH_INT(unsigned char); + ROBIN_HOOD_HASH_INT(char16_t); + ROBIN_HOOD_HASH_INT(char32_t); + ROBIN_HOOD_HASH_INT(wchar_t); + ROBIN_HOOD_HASH_INT(short); + ROBIN_HOOD_HASH_INT(unsigned short); + ROBIN_HOOD_HASH_INT(int); + ROBIN_HOOD_HASH_INT(unsigned int); + ROBIN_HOOD_HASH_INT(long); + ROBIN_HOOD_HASH_INT(long long); + ROBIN_HOOD_HASH_INT(unsigned long); + ROBIN_HOOD_HASH_INT(unsigned long long); +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +#endif + namespace detail { + +// using wrapper classes for hash and key_equal prevents the diamond problem when the same type is +// used. see https://stackoverflow.com/a/28771920/48181 + template + struct WrapHash : public T { + WrapHash() = default; + explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval()))) + : T(o) {} + }; + + template + struct WrapKeyEqual : public T { + WrapKeyEqual() = default; + explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval()))) + : T(o) {} + }; + +// A highly optimized hashmap implementation, using the Robin Hood algorithm. +// +// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but be +// about 2x faster in most cases and require much less allocations. +// +// This implementation uses the following memory layout: +// +// [Node, Node, ... Node | info, info, ... infoSentinel ] +// +// * Node: either a DataNode that directly has the std::pair as member, +// or a DataNode with a pointer to std::pair. Which DataNode representation to use +// depends on how fast the swap() operation is. Heuristically, this is automatically choosen based +// on sizeof(). there are always 2^n Nodes. +// +// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes. +// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the +// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it +// actually belongs to the previous position and was pushed out because that place is already +// taken. +// +// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the need +// for a idx +// variable. +// +// According to STL, order of templates has effect on throughput. That's why I've moved the boolean +// to the front. +// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/ + template + class Table + : public WrapHash, + public WrapKeyEqual, + detail::NodeAllocator< + typename std::conditional< + std::is_void::value, Key, + robin_hood::pair::type, T>>::type, + 4, 16384, IsFlat> { + public: + static constexpr bool is_flat = IsFlat; + static constexpr bool is_map = !std::is_void::value; + static constexpr bool is_set = !is_map; + + using key_type = Key; + using mapped_type = T; + using value_type = typename std::conditional< + is_set, Key, + robin_hood::pair::type, T>>::type; + using size_type = size_t; + using hasher = Hash; + using key_equal = KeyEqual; + using Self = Table; + + private: + static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, + "MaxLoadFactor100 needs to be >10 && < 100"); + + using WHash = WrapHash; + using WKeyEqual = WrapKeyEqual; + + // configuration defaults + + // make sure we have 8 elements, needed to quickly rehash mInfo + static constexpr size_t InitialNumElements = sizeof(uint64_t); + static constexpr uint32_t InitialInfoNumBits = 5; + static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; + static constexpr uint8_t InitialInfoHashShift = sizeof(size_t) * 8 - InitialInfoNumBits; + using DataPool = detail::NodeAllocator; + + // type needs to be wider than uint8_t. + using InfoType = uint32_t; + + // DataNode //////////////////////////////////////////////////////// + + // Primary template for the data node. We have special implementations for small and big + // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these on + // the heap so swap merely swaps a pointer. + template + class DataNode {}; + + // Small: just allocate on the stack. + template + class DataNode final { + public: + template + explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept( + noexcept(value_type(std::forward(args)...))) + : mData(std::forward(args)...) {} + + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode&& n) noexcept( + std::is_nothrow_move_constructible::value) + : mData(std::move(n.mData)) {} + + // doesn't do anything + void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {} + void destroyDoNotDeallocate() noexcept {} + + value_type const* operator->() const noexcept { + return &mData; + } + value_type* operator->() noexcept { + return &mData; + } + + const value_type& operator*() const noexcept { + return mData; + } + + value_type& operator*() noexcept { + return mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return mData.first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const + noexcept { + return mData.first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept { + return mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept { + return mData.second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept { + return mData.second; + } + + void swap(DataNode& o) noexcept( + noexcept(std::declval().swap(std::declval()))) { + mData.swap(o.mData); + } + + private: + value_type mData; + }; + + // big object: allocate on heap. + template + class DataNode { + public: + template + explicit DataNode(M& map, Args&&... args) + : mData(map.allocate()) { + ::new (static_cast(mData)) value_type(std::forward(args)...); + } + + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode&& n) noexcept + : mData(std::move(n.mData)) {} + + void destroy(M& map) noexcept { + // don't deallocate, just put it into list of datapool. + mData->~value_type(); + map.deallocate(mData); + } + + void destroyDoNotDeallocate() noexcept { + mData->~value_type(); + } + + value_type const* operator->() const noexcept { + return mData; + } + + value_type* operator->() noexcept { + return mData; + } + + const value_type& operator*() const { + return *mData; + } + + value_type& operator*() { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return mData->first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const + noexcept { + return mData->first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept { + return mData->second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept { + return mData->second; + } + + void swap(DataNode& o) noexcept { + using std::swap; + swap(mData, o.mData); + } + + private: + value_type* mData; + }; + + using Node = DataNode; + + // helpers for doInsert: extract first entry (only const required) + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept { + return n.getFirst(); + } + + // in case we have void mapped_type, we are not using a pair, thus we just route k through. + // No need to disable this because it's just not used if not applicable. + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept { + return k; + } + + // in case we have non-void mapped_type, we have a standard robin_hood::pair + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, key_type const&>::type + getFirstConst(value_type const& vt) const noexcept { + return vt.first; + } + + // Cloner ////////////////////////////////////////////////////////// + + template + struct Cloner; + + // fast path: Just copy data, without allocating anything. + template + struct Cloner { + void operator()(M const& source, M& target) const { + auto src = reinterpret_cast(source.mKeyVals); + auto tgt = reinterpret_cast(target.mKeyVals); + auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); + std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); + } + }; + + template + struct Cloner { + void operator()(M const& s, M& t) const { + auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1); + std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo); + + for (size_t i = 0; i < numElementsWithBuffer; ++i) { + if (t.mInfo[i]) { + ::new (static_cast(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]); + } + } + } + }; + + // Destroyer /////////////////////////////////////////////////////// + + template + struct Destroyer {}; + + template + struct Destroyer { + void nodes(M& m) const noexcept { + m.mNumElements = 0; + } + + void nodesDoNotDeallocate(M& m) const noexcept { + m.mNumElements = 0; + } + }; + + template + struct Destroyer { + void nodes(M& m) const noexcept { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { + if (0 != m.mInfo[idx]) { + Node& n = m.mKeyVals[idx]; + n.destroy(m); + n.~Node(); + } + } + } + + void nodesDoNotDeallocate(M& m) const noexcept { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { + if (0 != m.mInfo[idx]) { + Node& n = m.mKeyVals[idx]; + n.destroyDoNotDeallocate(); + n.~Node(); + } + } + } + }; + + // Iter //////////////////////////////////////////////////////////// + + struct fast_forward_tag {}; + + // generic iterator for both const_iterator and iterator. + template + // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions) + class Iter { + private: + using NodePtr = typename std::conditional::type; + + public: + using difference_type = std::ptrdiff_t; + using value_type = typename Self::value_type; + using reference = typename std::conditional::type; + using pointer = typename std::conditional::type; + using iterator_category = std::forward_iterator_tag; + + // default constructed iterator can be compared to itself, but WON'T return true when + // compared to end(). + Iter() = default; + + // Rule of zero: nothing specified. The conversion constructor is only enabled for iterator + // to const_iterator, so it doesn't accidentally work as a copy ctor. + + // Conversion constructor from iterator to const_iterator. + template ::type> + // NOLINTNEXTLINE(hicpp-explicit-conversions) + Iter(Iter const& other) noexcept + : mKeyVals(other.mKeyVals) + , mInfo(other.mInfo) {} + + Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept + : mKeyVals(valPtr) + , mInfo(infoPtr) {} + + Iter(NodePtr valPtr, uint8_t const* infoPtr, + fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept + : mKeyVals(valPtr) + , mInfo(infoPtr) { + fastForward(); + } + + template ::type> + Iter& operator=(Iter const& other) noexcept { + mKeyVals = other.mKeyVals; + mInfo = other.mInfo; + return *this; + } + + // prefix increment. Undefined behavior if we are at end()! + Iter& operator++() noexcept { + mInfo++; + mKeyVals++; + fastForward(); + return *this; + } + + reference operator*() const { + return **mKeyVals; + } + + pointer operator->() const { + return &**mKeyVals; + } + + template + bool operator==(Iter const& o) const noexcept { + return mKeyVals == o.mKeyVals; + } + + template + bool operator!=(Iter const& o) const noexcept { + return mKeyVals != o.mKeyVals; + } + + private: + // fast forward to the next non-free info byte + void fastForward() noexcept { + int inc; + do { + auto const n = detail::unaligned_load(mInfo); +#if ROBIN_HOOD(LITTLE_ENDIAN) + inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; +#else + inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; +#endif + mInfo += inc; + mKeyVals += inc; + } while (inc == static_cast(sizeof(size_t))); + } + + friend class Table; + NodePtr mKeyVals{nullptr}; + uint8_t const* mInfo{nullptr}; + }; + + //////////////////////////////////////////////////////////////////// + + // highly performance relevant code. + // Lower bits are used for indexing into the array (2^n size) + // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. + template + void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const { + // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as an + // additional mixing step. This serves as a bad hash prevention, if the given data is badly + // mixed. + using Mix = + typename std::conditional, hasher>::value, + ::robin_hood::detail::identity_hash, + ::robin_hood::hash>::type; + *idx = Mix{}(WHash::operator()(key)); + + *info = mInfoInc + static_cast(*idx >> mInfoHashShift); + *idx &= mMask; + } + + // forwards the index by one, wrapping around at the end + void next(InfoType* info, size_t* idx) const noexcept { + *idx = *idx + 1; + *info += mInfoInc; + } + + void nextWhileLess(InfoType* info, size_t* idx) const noexcept { + // unrolling this by hand did not bring any speedups. + while (*info < mInfo[*idx]) { + next(info, idx); + } + } + + // Shift everything up by one element. Tries to move stuff around. + void + shiftUp(size_t startIdx, + size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable::value) { + auto idx = startIdx; + ::new (static_cast(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1])); + while (--idx != insertion_idx) { + mKeyVals[idx] = std::move(mKeyVals[idx - 1]); + } + + idx = startIdx; + while (idx != insertion_idx) { + ROBIN_HOOD_COUNT(shiftUp); + mInfo[idx] = static_cast(mInfo[idx - 1] + mInfoInc); + if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + --idx; + } + } + + void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable::value) { + // until we find one that is either empty or has zero offset. + // TODO(martinus) we don't need to move everything, just the last one for the same bucket. + mKeyVals[idx].destroy(*this); + + // until we find one that is either empty or has zero offset. + while (mInfo[idx + 1] >= 2 * mInfoInc) { + ROBIN_HOOD_COUNT(shiftDown); + mInfo[idx] = static_cast(mInfo[idx + 1] - mInfoInc); + mKeyVals[idx] = std::move(mKeyVals[idx + 1]); + ++idx; + } + + mInfo[idx] = 0; + // don't destroy, we've moved it + // mKeyVals[idx].destroy(*this); + mKeyVals[idx].~Node(); + } + + // copy of find(), except that it returns iterator instead of const_iterator. + template + ROBIN_HOOD(NODISCARD) + size_t findIdx(Other const& key) const { + size_t idx; + InfoType info; + keyToIdx(key, &idx, &info); + + do { + // unrolling this twice gives a bit of a speedup. More unrolling did not help. + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { + return idx; + } + next(&info, &idx); + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { + return idx; + } + next(&info, &idx); + } while (info <= mInfo[idx]); + + // nothing found! + return mMask == 0 ? 0 + : static_cast(std::distance( + mKeyVals, reinterpret_cast_no_cast_align_warning(mInfo))); + } + + void cloneData(const Table& o) { + Cloner()(o, *this); + } + + // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized. + // @return index where the element was created + size_t insert_move(Node&& keyval) { + // we don't retry, fail if overflowing + // don't need to check max num elements + if (0 == mMaxNumElementsAllowed && !try_increase_info()) { + throwOverflowError(); // impossible to reach LCOV_EXCL_LINE + } + + size_t idx; + InfoType info; + keyToIdx(keyval.getFirst(), &idx, &info); + + // skip forward. Use <= because we are certain that the element is not there. + while (info <= mInfo[idx]) { + idx = idx + 1; + info += mInfoInc; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = static_cast(info); + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) { + next(&info, &idx); + } + + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) { + ::new (static_cast(&l)) Node(std::move(keyval)); + } else { + shiftUp(idx, insertion_idx); + l = std::move(keyval); + } + + // put at empty spot + mInfo[insertion_idx] = insertion_info; + + ++mNumElements; + return insertion_idx; + } + + public: + using iterator = Iter; + using const_iterator = Iter; + + // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. This + // tremendously speeds up ctor & dtor of a map that never receives an element. The penalty is + // payed at the first insert, and not before. Lookup of this empty map works because everybody + // points to DummyInfoByte::b. parameter bucket_count is dictated by the standard, but we can + // ignore it. + explicit Table(size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && + noexcept(KeyEqual(equal))) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this); + } + + template + Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, + const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this); + insert(first, last); + } + + Table(std::initializer_list initlist, + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this); + insert(initlist.begin(), initlist.end()); + } + + Table(Table&& o) noexcept + : WHash(std::move(static_cast(o))) + , WKeyEqual(std::move(static_cast(o))) + , DataPool(std::move(static_cast(o))) { + ROBIN_HOOD_TRACE(this); + if (o.mMask) { + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + // set other's mask to 0 so its destructor won't do anything + o.init(); + } + } + + Table& operator=(Table&& o) noexcept { + ROBIN_HOOD_TRACE(this); + if (&o != this) { + if (o.mMask) { + // only move stuff if the other map actually has some data + destroy(); + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + WHash::operator=(std::move(static_cast(o))); + WKeyEqual::operator=(std::move(static_cast(o))); + DataPool::operator=(std::move(static_cast(o))); + + o.init(); + + } else { + // nothing in the other map => just clear us. + clear(); + } + } + return *this; + } + + Table(const Table& o) + : WHash(static_cast(o)) + , WKeyEqual(static_cast(o)) + , DataPool(static_cast(o)) { + ROBIN_HOOD_TRACE(this); + if (!o.empty()) { + // not empty: create an exact copy. it is also possible to just iterate through all + // elements and insert them, but copying is probably faster. + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + mKeyVals = static_cast(detail::assertNotNull( + malloc(calcNumBytesTotal(numElementsWithBuffer)))); + // no need for calloc because clonData does memcpy + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + } + } + + // Creates a copy of the given map. Copy constructor of each entry is used. + Table& operator=(Table const& o) { + ROBIN_HOOD_TRACE(this); + if (&o == this) { + // prevent assigning of itself + return *this; + } + + // we keep using the old allocator and not assign the new one, because we want to keep the + // memory available. when it is the same size. + if (o.empty()) { + if (0 == mMask) { + // nothing to do, we are empty too + return *this; + } + + // not empty: destroy what we have there + // clear also resets mInfo to 0, that's sometimes not necessary. + destroy(); + init(); + WHash::operator=(static_cast(o)); + WKeyEqual::operator=(static_cast(o)); + DataPool::operator=(static_cast(o)); + + return *this; + } + + // clean up old stuff + Destroyer::value>{}.nodes(*this); + + if (mMask != o.mMask) { + // no luck: we don't have the same array size allocated, so we need to realloc. + if (0 != mMask) { + // only deallocate if we actually have data! + free(mKeyVals); + } + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + mKeyVals = static_cast(detail::assertNotNull( + malloc(calcNumBytesTotal(numElementsWithBuffer)))); + + // no need for calloc here because cloneData performs a memcpy. + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + // sentinel is set in cloneData + } + WHash::operator=(static_cast(o)); + WKeyEqual::operator=(static_cast(o)); + DataPool::operator=(static_cast(o)); + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + + return *this; + } + + // Swaps everything between the two maps. + void swap(Table& o) { + ROBIN_HOOD_TRACE(this); + using std::swap; + swap(o, *this); + } + + // Clears all data, without resizing. + void clear() { + ROBIN_HOOD_TRACE(this); + if (empty()) { + // don't do anything! also important because we don't want to write to DummyInfoByte::b, + // even though we would just write 0 to it. + return; + } + + Destroyer::value>{}.nodes(*this); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + // clear everything, then set the sentinel again + uint8_t const z = 0; + std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z); + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // Destroys the map and all it's contents. + ~Table() { + ROBIN_HOOD_TRACE(this); + destroy(); + } + + // Checks if both tables contain the same entries. Order is irrelevant. + bool operator==(const Table& other) const { + ROBIN_HOOD_TRACE(this); + if (other.size() != size()) { + return false; + } + for (auto const& otherEntry : other) { + if (!has(otherEntry)) { + return false; + } + } + + return true; + } + + bool operator!=(const Table& other) const { + ROBIN_HOOD_TRACE(this); + return !operator==(other); + } + + template + typename std::enable_if::value, Q&>::type operator[](const key_type& key) { + ROBIN_HOOD_TRACE(this); + return doCreateByKey(key); + } + + template + typename std::enable_if::value, Q&>::type operator[](key_type&& key) { + ROBIN_HOOD_TRACE(this); + return doCreateByKey(std::move(key)); + } + + template + void insert(Iter first, Iter last) { + for (; first != last; ++first) { + // value_type ctor needed because this might be called with std::pair's + insert(value_type(*first)); + } + } + + template + std::pair emplace(Args&&... args) { + ROBIN_HOOD_TRACE(this); + Node n{*this, std::forward(args)...}; + auto r = doInsert(std::move(n)); + if (!r.second) { + // insertion not possible: destroy node + // NOLINTNEXTLINE(bugprone-use-after-move) + n.destroy(*this); + } + return r; + } + + std::pair insert(const value_type& keyval) { + ROBIN_HOOD_TRACE(this); + return doInsert(keyval); + } + + std::pair insert(value_type&& keyval) { + return doInsert(std::move(keyval)); + } + + // Returns 1 if key is found, 0 otherwise. + size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { + return 1; + } + return 0; + } + + bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + return 1U == count(key); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q&>::type at(key_type const& key) { + ROBIN_HOOD_TRACE(this); + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { + doThrow("key not found"); + } + return kv->getSecond(); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q const&>::type at(key_type const& key) const { + ROBIN_HOOD_TRACE(this); + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { + doThrow("key not found"); + } + return kv->getSecond(); + } + + const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template + const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const { + ROBIN_HOOD_TRACE(this); + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator find(const key_type& key) { + ROBIN_HOOD_TRACE(this); + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template + iterator find(const OtherKey& key, is_transparent_tag /*unused*/) { + ROBIN_HOOD_TRACE(this); + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator begin() { + ROBIN_HOOD_TRACE(this); + if (empty()) { + return end(); + } + return iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + const_iterator begin() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + return cbegin(); + } + const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + if (empty()) { + return cend(); + } + return const_iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + + iterator end() { + ROBIN_HOOD_TRACE(this); + // no need to supply valid info pointer: end() must not be dereferenced, and only node + // pointer is compared. + return iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; + } + const_iterator end() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + return cend(); + } + const_iterator cend() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + return const_iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; + } + + iterator erase(const_iterator pos) { + ROBIN_HOOD_TRACE(this); + // its safe to perform const cast here + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return erase(iterator{const_cast(pos.mKeyVals), const_cast(pos.mInfo)}); + } + + // Erases element at pos, returns iterator to the next element. + iterator erase(iterator pos) { + ROBIN_HOOD_TRACE(this); + // we assume that pos always points to a valid entry, and not end(). + auto const idx = static_cast(pos.mKeyVals - mKeyVals); + + shiftDown(idx); + --mNumElements; + + if (*pos.mInfo) { + // we've backward shifted, return this again + return pos; + } + + // no backward shift, return next element + return ++pos; + } + + size_t erase(const key_type& key) { + ROBIN_HOOD_TRACE(this); + size_t idx; + InfoType info; + keyToIdx(key, &idx, &info); + + // check while info matches with the source idx + do { + if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + shiftDown(idx); + --mNumElements; + return 1; + } + next(&info, &idx); + } while (info <= mInfo[idx]); + + // nothing found to delete + return 0; + } + + // reserves space for the specified number of elements. Makes sure the old data fits. + // exactly the same as reserve(c). + void rehash(size_t c) { + reserve(c); + } + + // reserves space for the specified number of elements. Makes sure the old data fits. + // Exactly the same as resize(c). Use resize(0) to shrink to fit. + void reserve(size_t c) { + ROBIN_HOOD_TRACE(this); + auto const minElementsAllowed = (std::max)(c, mNumElements); + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { + throwOverflowError(); + } + + rehashPowerOfTwo(newSize); + } + + size_type size() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + return mNumElements; + } + + size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + return static_cast(-1); + } + + ROBIN_HOOD(NODISCARD) bool empty() const noexcept { + ROBIN_HOOD_TRACE(this); + return 0 == mNumElements; + } + + float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + return MaxLoadFactor100 / 100.0F; + } + + // Average number of elements per bucket. Since we allow only 1 per bucket + float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this); + return static_cast(size()) / static_cast(mMask + 1); + } + + ROBIN_HOOD(NODISCARD) size_t mask() const noexcept { + ROBIN_HOOD_TRACE(this); + return mMask; + } + + ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept { + if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits::max)() / 100)) { + return maxElements * MaxLoadFactor100 / 100; + } + + // we might be a bit inprecise, but since maxElements is quite large that doesn't matter + return (maxElements / 100) * MaxLoadFactor100; + } + + ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept { + // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load + // 64bit types. + return numElements + sizeof(uint64_t); + } + + ROBIN_HOOD(NODISCARD) + size_t calcNumElementsWithBuffer(size_t numElements) const noexcept { + auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements); + return numElements + (std::min)(maxNumElementsAllowed, (static_cast(0xFF))); + } + + // calculation only allowed for 2^n values + ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const { +#if ROBIN_HOOD(BITNESS) == 64 + return numElements * sizeof(Node) + calcNumBytesInfo(numElements); +#else + // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows. + auto const ne = static_cast(numElements); + auto const s = static_cast(sizeof(Node)); + auto const infos = static_cast(calcNumBytesInfo(numElements)); + + auto const total64 = ne * s + infos; + auto const total = static_cast(total64); + + if (ROBIN_HOOD_UNLIKELY(static_cast(total) != total64)) { + throwOverflowError(); + } + return total; +#endif + } + + private: + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this); + auto it = find(e.first); + return it != end() && it->second == e.second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this); + return find(e) != end(); + } + + // reserves space for at least the specified number of elements. + // only works if numBuckets if power of two + void rehashPowerOfTwo(size_t numBuckets) { + ROBIN_HOOD_TRACE(this); + + Node* const oldKeyVals = mKeyVals; + uint8_t const* const oldInfo = mInfo; + + const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + // resize operation: move stuff + init_data(numBuckets); + if (oldMaxElementsWithBuffer > 1) { + for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) { + if (oldInfo[i] != 0) { + insert_move(std::move(oldKeyVals[i])); + // destroy the node but DON'T destroy the data. + oldKeyVals[i].~Node(); + } + } + + // don't destroy old data: put it into the pool instead + DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer)); + } + } + + ROBIN_HOOD(NOINLINE) void throwOverflowError() const { +#if ROBIN_HOOD(HAS_EXCEPTIONS) + throw std::overflow_error("robin_hood::map overflow"); +#else + abort(); +#endif + } + + void init_data(size_t max_elements) { + mNumElements = 0; + mMask = max_elements - 1; + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements); + + // calloc also zeroes everything + mKeyVals = reinterpret_cast(detail::assertNotNull( + calloc(1, calcNumBytesTotal(numElementsWithBuffer)))); + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + + // set sentinel + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + template + typename std::enable_if::value, Q&>::type doCreateByKey(Arg&& key) { + while (true) { + size_t idx; + InfoType info; + keyToIdx(key, &idx, &info); + nextWhileLess(&info, &idx); + + // while we potentially have a match. Can't do a do-while here because when mInfo is 0 + // we don't want to skip forward + while (info == mInfo[idx]) { + if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + // key already exists, do not insert. + return mKeyVals[idx].getSecond(); + } + next(&info, &idx); + } + + // unlikely that this evaluates to true + if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { + increase_size(); + continue; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = info; + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) { + next(&info, &idx); + } + + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) { + // put at empty spot. This forwards all arguments into the node where the object is + // constructed exactly where it is needed. + ::new (static_cast(&l)) + Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), std::forward_as_tuple()); + } else { + shiftUp(idx, insertion_idx); + l = Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), std::forward_as_tuple()); + } + + // mKeyVals[idx].getFirst() = std::move(key); + mInfo[insertion_idx] = static_cast(insertion_info); + + ++mNumElements; + return mKeyVals[insertion_idx].getSecond(); + } + } + + // This is exactly the same code as operator[], except for the return values + template + std::pair doInsert(Arg&& keyval) { + while (true) { + size_t idx; + InfoType info; + keyToIdx(getFirstConst(keyval), &idx, &info); + nextWhileLess(&info, &idx); + + // while we potentially have a match + while (info == mInfo[idx]) { + if (WKeyEqual::operator()(getFirstConst(keyval), mKeyVals[idx].getFirst())) { + // key already exists, do NOT insert. + // see http://en.cppreference.com/w/cpp/container/unordered_map/insert + return std::make_pair(iterator(mKeyVals + idx, mInfo + idx), + false); + } + next(&info, &idx); + } + + // unlikely that this evaluates to true + if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { + increase_size(); + continue; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = info; + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) { + next(&info, &idx); + } + + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) { + ::new (static_cast(&l)) Node(*this, std::forward(keyval)); + } else { + shiftUp(idx, insertion_idx); + l = Node(*this, std::forward(keyval)); + } + + // put at empty spot + mInfo[insertion_idx] = static_cast(insertion_info); + + ++mNumElements; + return std::make_pair(iterator(mKeyVals + insertion_idx, mInfo + insertion_idx), true); + } + } + + bool try_increase_info() { + ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements + << ", maxNumElementsAllowed=" + << calcMaxNumElementsAllowed(mMask + 1)); + if (mInfoInc <= 2) { + // need to be > 2 so that shift works (otherwise undefined behavior!) + return false; + } + // we got space left, try to make info smaller + mInfoInc = static_cast(mInfoInc >> 1U); + + // remove one bit of the hash, leaving more space for the distance info. + // This is extremely fast because we can operate on 8 bytes at once. + ++mInfoHashShift; + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + for (size_t i = 0; i < numElementsWithBuffer; i += 8) { + auto val = unaligned_load(mInfo + i); + val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); + std::memcpy(mInfo + i, &val, sizeof(val)); + } + // update sentinel, which might have been cleared out! + mInfo[numElementsWithBuffer] = 1; + + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + return true; + } + + void increase_size() { + // nothing allocated yet? just allocate InitialNumElements + if (0 == mMask) { + init_data(InitialNumElements); + return; + } + + auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + if (mNumElements < maxNumElementsAllowed && try_increase_info()) { + return; + } + + ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" + << maxNumElementsAllowed << ", load=" + << (static_cast(mNumElements) * 100.0 / + (static_cast(mMask) + 1))); + // it seems we have a really bad hash function! don't try to resize again + if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { + throwOverflowError(); + } + + rehashPowerOfTwo((mMask + 1) * 2); + } + + void destroy() { + if (0 == mMask) { + // don't deallocate! + return; + } + + Destroyer::value>{} + .nodesDoNotDeallocate(*this); + + // This protection against not deleting mMask shouldn't be needed as it's sufficiently + // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise + // reports a compile error: attempt to free a non-heap object ‘fm’ + // [-Werror=free-nonheap-object] + if (mKeyVals != reinterpret_cast(&mMask)) { + free(mKeyVals); + } + } + + void init() noexcept { + mKeyVals = reinterpret_cast(&mMask); + mInfo = reinterpret_cast(&mMask); + mNumElements = 0; + mMask = 0; + mMaxNumElementsAllowed = 0; + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // members are sorted so no padding occurs + Node* mKeyVals = reinterpret_cast(&mMask); // 8 byte 8 + uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 16 + size_t mNumElements = 0; // 8 byte 24 + size_t mMask = 0; // 8 byte 32 + size_t mMaxNumElementsAllowed = 0; // 8 byte 40 + InfoType mInfoInc = InitialInfoInc; // 4 byte 44 + InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48 + // 16 byte 56 if NodeAllocator + }; + + } // namespace detail + +// map + + template , + typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> + using unordered_flat_map = detail::Table; + + template , + typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> + using unordered_node_map = detail::Table; + + template , + typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> + using unordered_map = + detail::Table) <= sizeof(size_t) * 6 && + std::is_nothrow_move_constructible>::value && + std::is_nothrow_move_assignable>::value, + MaxLoadFactor100, Key, T, Hash, KeyEqual>; + +// set + + template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> + using unordered_flat_set = detail::Table; + + template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> + using unordered_node_set = detail::Table; + + template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> + using unordered_set = detail::Table::value && + std::is_nothrow_move_assignable::value, + MaxLoadFactor100, Key, void, Hash, KeyEqual>; + +} // namespace robin_hood + +#endif \ No newline at end of file diff --git a/src/pubkey.h b/src/pubkey.h index 471c8dfc5..9ca51e1dd 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -40,9 +40,9 @@ namespace std { template <> struct hash { - std::size_t operator()(const CKeyID& k) const + size_t operator()(const CKeyID& k) const { - return *reinterpret_cast(k.begin()); + return *reinterpret_cast(k.begin()); } }; } diff --git a/src/script/standard.h b/src/script/standard.h index d2ab00a11..59bf18805 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -31,9 +31,9 @@ namespace std { template <> struct hash { - std::size_t operator()(const CScriptID& k) const + size_t operator()(const CScriptID& k) const { - return *reinterpret_cast(k.begin()); + return *reinterpret_cast(k.begin()); } }; } diff --git a/src/validation.h b/src/validation.h index 4083ced90..df116139c 100644 --- a/src/validation.h +++ b/src/validation.h @@ -17,6 +17,7 @@ #include #include // For CMessageHeader::MessageStartChars #include +#include #include