fixed slow loading of block index, increased wallet flush period

src/claimtrie/blob.cpp

@@ -141,25 +141,25 @@ std::string CBaseBlob<BITS>::ToString() const
 template<uint32_t BITS>
 uint8_t* CBaseBlob<BITS>::begin() noexcept
 {
-    return data.begin();
+    return data.data();
 }
 
 template<uint32_t BITS>
 const uint8_t* CBaseBlob<BITS>::begin() const noexcept
 {
-    return data.begin();
+    return data.data();
 }
 
 template<uint32_t BITS>
 uint8_t* CBaseBlob<BITS>::end() noexcept
 {
-    return data.end();
+    return data.data() + data.size();
 }
 
 template<uint32_t BITS>
 const uint8_t* CBaseBlob<BITS>::end() const noexcept
 {
-    return data.end();
+    return data.data() + data.size();
 }
 
 template<uint32_t BITS>

src/init.cpp

@@ -1213,6 +1213,8 @@ bool AppInitLockDataDirectory()
     return true;
 }
 
+extern CChainState g_chainstate;
+
 bool AppInitMain()
 {
     const CChainParams& chainparams = Params();
@@ -1481,7 +1483,7 @@ bool AppInitMain()
 
                 // If the loaded chain has a wrong genesis, bail out immediately
                 // (we're likely using a testnet datadir, or the other way around).
-                if (!mapBlockIndex.empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) {
+                if (!g_chainstate.mapBlockIndex.empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) {
                     return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?"));
                 }
 
@@ -1711,7 +1713,6 @@ bool AppInitMain()
     //// debug print
     {
         LOCK(cs_main);
-        LogPrintf("mapBlockIndex.size() = %u\n", mapBlockIndex.size());
         chain_active_height = chainActive.Height();
     }
     LogPrintf("nBestHeight = %d\n", chain_active_height);

src/interfaces/wallet.cpp

@@ -88,8 +88,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx)
 WalletTxStatus MakeWalletTxStatus(const CWalletTx& wtx)
 {
     WalletTxStatus result;
-    auto mi = ::mapBlockIndex.find(wtx.hashBlock);
-    CBlockIndex* block = mi != ::mapBlockIndex.end() ? mi->second : nullptr;
+    CBlockIndex* block = LookupBlockIndex(wtx.hashBlock);
     result.block_height = (block ? block->nHeight : std::numeric_limits<int>::max());
     result.blocks_to_maturity = wtx.GetBlocksToMaturity();
     result.depth_in_main_chain = wtx.GetDepthInMainChain();

src/rpc/blockchain.cpp

@@ -1299,6 +1299,8 @@ struct CompareBlocksByHeight
     }
 };
 
+extern CChainState g_chainstate;
+
 static UniValue getchaintips(const JSONRPCRequest& request)
 {
     if (request.fHelp || request.params.size() != 0)
@@ -1347,7 +1349,7 @@ static UniValue getchaintips(const JSONRPCRequest& request)
     std::set<const CBlockIndex*> setOrphans;
     std::set<const CBlockIndex*> setPrevs;
 
-    for (const std::pair<const uint256, CBlockIndex*>& item : mapBlockIndex)
+    for (const auto& item : g_chainstate.mapBlockIndex)
     {
         if (!chainActive.Contains(item.second)) {
             setOrphans.insert(item.second);

src/rpc/claimtrie.cpp

@@ -33,16 +33,15 @@ void ParseClaimtrieId(const UniValue& v, std::string& claimId, const std::string
 
 static CBlockIndex* BlockHashIndex(const uint256& blockHash)
 {
-    AssertLockHeld(cs_main);
+    auto index = LookupBlockIndex(blockHash);
 
-    if (mapBlockIndex.count(blockHash) == 0)
+    if (index == nullptr)
         throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
 
-    CBlockIndex* pblockIndex = mapBlockIndex[blockHash];
-    if (!chainActive.Contains(pblockIndex))
+    if (!chainActive.Contains(index))
         throw JSONRPCError(RPC_INTERNAL_ERROR, "Block not in main chain");
 
-    return pblockIndex;
+    return index;
 }
 
 #define MAX_RPC_BLOCK_DECREMENTS 500

src/txdb.cpp

@@ -361,29 +361,33 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams,
 {
     auto query = db << "SELECT hash, prevHash, height, file, dataPos, undoPos, txCount, "
                        "version, rootTxHash, rootTrieHash, time, bits, nonce, status "
-                       "FROM block_info"; // insertBlockIndex puts them in order
+                       "FROM block_info ORDER BY height";
 
-    // Load mapBlockIndex
     for (auto&& row: query) {
-        boost::this_thread::interruption_point();
         // Construct block index object
         uint256 hash, prevHash;
         row >> hash >> prevHash;
         CBlockIndex* pindexNew    = insertBlockIndex(hash);
         pindexNew->pprev          = insertBlockIndex(prevHash);
 
-        row >> pindexNew->nHeight;
-        row >> pindexNew->nFile;
-        row >> pindexNew->nDataPos;
-        row >> pindexNew->nUndoPos;
-        row >> pindexNew->nTx;
-        row >> pindexNew->nVersion;
-        row >> pindexNew->hashMerkleRoot;
-        row >> pindexNew->hashClaimTrie;
-        row >> pindexNew->nTime;
-        row >> pindexNew->nBits;
-        row >> pindexNew->nNonce;
-        row >> pindexNew->nStatus;
+        row >> pindexNew->nHeight
+            >> pindexNew->nFile
+            >> pindexNew->nDataPos
+            >> pindexNew->nUndoPos
+            >> pindexNew->nTx
+            >> pindexNew->nVersion
+            >> pindexNew->hashMerkleRoot
+            >> pindexNew->hashClaimTrie
+            >> pindexNew->nTime
+            >> pindexNew->nBits
+            >> pindexNew->nNonce
+            >> pindexNew->nStatus;
+
+        if ((pindexNew->nHeight & 0x3ff) == 0x3ff) { // don't check for shutdown on every single block
+            boost::this_thread::interruption_point();
+            if (ShutdownRequested())
+                return false;
+        }
 
         pindexNew->nChainTx = pindexNew->pprev ? pindexNew->pprev->nChainTx + pindexNew->nTx : pindexNew->nTx;
 

src/validation.cpp

@@ -62,7 +62,6 @@
 CChainState g_chainstate;
 CCriticalSection cs_main;
 
-BlockMap& mapBlockIndex = g_chainstate.mapBlockIndex;
 CChain& chainActive = g_chainstate.chainActive;
 CBlockIndex *pindexBestHeader = nullptr;
 CWaitableCriticalSection g_best_block_mutex;
@@ -3662,7 +3661,7 @@ void PruneOneBlockFile(const int fileNumber)
 {
     LOCK(cs_LastBlockFile);
 
-    for (const auto& entry : mapBlockIndex) {
+    for (const auto& entry : g_chainstate.mapBlockIndex) {
         CBlockIndex* pindex = entry.second;
         if (pindex->nFile == fileNumber) {
             pindex->nStatus &= ~BLOCK_HAVE_DATA;
@@ -3857,18 +3856,18 @@ CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash)
 {
     AssertLockHeld(cs_main);
 
+    // Return existing
+    auto mi = mapBlockIndex.find(hash);
+    if (mi != mapBlockIndex.end())
+        return mi->second;
+
     if (hash.IsNull())
         return nullptr;
 
-    // Return existing
-    BlockMap::iterator mi = mapBlockIndex.find(hash);
-    if (mi != mapBlockIndex.end())
-        return (*mi).second;
-
     // Create new
     CBlockIndex* pindexNew = new CBlockIndex();
     mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first;
-    pindexNew->phashBlock = &((*mi).first);
+    pindexNew->phashBlock = &(mi->first);
 
     return pindexNew;
 }
@@ -3950,14 +3949,14 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE
     // Check presence of blk files
     LogPrintf("Checking all blk files are present...\n");
     std::set<int> setBlkDataFiles;
-    for (const std::pair<const uint256, CBlockIndex*>& item : mapBlockIndex)
+    for (const std::pair<const uint256, CBlockIndex*>& item : g_chainstate.mapBlockIndex)
     {
         CBlockIndex* pindex = item.second;
         if (pindex->nStatus & BLOCK_HAVE_DATA) {
             setBlkDataFiles.insert(pindex->nFile);
         }
     }
-    for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++)
+    for (auto it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); ++it)
     {
         CDiskBlockPos pos(*it, 0);
         if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) {
@@ -3984,7 +3983,7 @@ bool LoadChainTip(const CChainParams& chainparams)
 
     if (chainActive.Tip() && chainActive.Tip()->GetBlockHash() == pcoinsTip->GetBestBlock()) return true;
 
-    if (pcoinsTip->GetBestBlock().IsNull() && mapBlockIndex.size() == 1) {
+    if (pcoinsTip->GetBestBlock().IsNull() && g_chainstate.mapBlockIndex.size() == 1) {
         // In case we just added the genesis block, connect it now, so
         // that we always have a chainActive.Tip() when we return.
         LogPrintf("%s: Connecting genesis block...\n", __func__);
@@ -4344,10 +4343,10 @@ void UnloadBlockIndex()
         warningcache[b].clear();
     }
 
-    for (BlockMap::value_type& entry : mapBlockIndex) {
+    for (BlockMap::value_type& entry : g_chainstate.mapBlockIndex) {
         delete entry.second;
     }
-    mapBlockIndex.clear();
+    g_chainstate.mapBlockIndex.clear();
     fHavePruned = false;
 
     g_chainstate.UnloadBlockIndex();
@@ -4360,7 +4359,7 @@ bool LoadBlockIndex(const CChainParams& chainparams)
     if (!fReindex) {
         bool ret = LoadBlockIndexDB(chainparams);
         if (!ret) return false;
-        needs_init = mapBlockIndex.empty();
+        needs_init = g_chainstate.mapBlockIndex.empty();
 
         if (needs_init) {
             auto blockDir = GetDataDir() / "blocks";
@@ -4899,15 +4898,21 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin
     return pindex->nChainTx / fTxTotal;
 }
 
+CBlockIndex *LookupBlockIndex(const uint256 &hash) {
+    AssertLockHeld(cs_main);
+    auto it = g_chainstate.mapBlockIndex.find(hash);
+    return it == g_chainstate.mapBlockIndex.end() ? nullptr : it->second;
+}
+
 class CMainCleanup
 {
 public:
     CMainCleanup() {}
     ~CMainCleanup() {
         // block headers
-        BlockMap::iterator it1 = mapBlockIndex.begin();
-        for (; it1 != mapBlockIndex.end(); it1++)
+        auto it1 = g_chainstate.mapBlockIndex.begin();
+        for (; it1 != g_chainstate.mapBlockIndex.end(); ++it1)
             delete (*it1).second;
-        mapBlockIndex.clear();
+        g_chainstate.mapBlockIndex.clear();
     }
 } instance_of_cmaincleanup;

src/validation.h

@@ -139,18 +139,12 @@ static const bool DEFAULT_PEERBLOOMFILTERS = true;
 /** Default for -stopatheight */
 static const int DEFAULT_STOPATHEIGHT = 0;
 
-struct BlockHasher
-{
-    size_t operator()(const uint256& hash) const { return GetCheapHash(hash); }
-};
-
 extern CScript COINBASE_FLAGS;
 extern CCriticalSection cs_main;
 extern CBlockPolicyEstimator feeEstimator;
 extern CTxMemPool mempool;
 extern std::atomic_bool g_is_mempool_loaded;
-typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap;
-extern BlockMap& mapBlockIndex;
+typedef std::map<uint256, CBlockIndex*> BlockMap;
 extern uint64_t nLastBlockTx;
 extern uint64_t nLastBlockWeight;
 extern const std::string strMessageMagic;
@@ -465,7 +459,7 @@ public:
 
     /**
      * If a block header hasn't already been seen, call CheckBlockHeader on it, ensure
-     * that it doesn't descend from an invalid block, and then add it to mapBlockIndex.
+         * that it doesn't descend from an invalid block, and then add it to mapBlockIndex.
      */
     bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
@@ -597,12 +591,7 @@ public:
 /** Replay blocks that aren't fully applied to the database. */
 bool ReplayBlocks(const CChainParams& params, CCoinsView* view);
 
-inline CBlockIndex* LookupBlockIndex(const uint256& hash)
-{
-    AssertLockHeld(cs_main);
-    BlockMap::const_iterator it = mapBlockIndex.find(hash);
-    return it == mapBlockIndex.end() ? nullptr : it->second;
-}
+CBlockIndex* LookupBlockIndex(const uint256& hash);
 
 /** Find the last common block between the parameter chain and a locator. */
 CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator);

src/wallet/init.cpp

@@ -245,7 +245,7 @@ void WalletInit::Start(CScheduler& scheduler) const
     }
 
     // Run a thread to flush wallet periodically
-    scheduler.scheduleEvery(MaybeCompactWalletDB, 500);
+    scheduler.scheduleEvery(MaybeCompactWalletDB, 1000);
 }
 
 void WalletInit::Flush() const

src/wallet/rpcwallet.cpp

@@ -765,23 +765,19 @@ void ListNameClaims(const CWalletTx& wtx, CWallet* const pwallet, const std::str
 
         CClaimTrieCache trieCache(pclaimTrie);
 
-        auto it = mapBlockIndex.find(wtx.hashBlock);
-        if (it != mapBlockIndex.end())
+        auto pindex = LookupBlockIndex(wtx.hashBlock);
+        if (pindex)
         {
-            CBlockIndex* pindex = it->second;
-            if (pindex)
+            entry.pushKV("height", pindex->nHeight);
+            entry.pushKV("expiration height", pindex->nHeight + trieCache.expirationTime());
+            if (pindex->nHeight + trieCache.expirationTime() > chainActive.Height())
             {
-                entry.pushKV("height", pindex->nHeight);
-                entry.pushKV("expiration height", pindex->nHeight + trieCache.expirationTime());
-                if (pindex->nHeight + trieCache.expirationTime() > chainActive.Height())
-                {
-                    entry.pushKV("expired", false);
-                    entry.pushKV("blocks to expiration", pindex->nHeight + trieCache.expirationTime() - chainActive.Height());
-                }
-                else
-                {
-                    entry.pushKV("expired", true);
-                }
+                entry.pushKV("expired", false);
+                entry.pushKV("blocks to expiration", pindex->nHeight + trieCache.expirationTime() - chainActive.Height());
+            }
+            else
+            {
+                entry.pushKV("expired", true);
             }
         }
         entry.pushKV("confirmations", wtx.GetDepthInMainChain());

src/wallet/rpcwallet.h

@@ -5,6 +5,7 @@
 #ifndef BITCOIN_WALLET_RPCWALLET_H
 #define BITCOIN_WALLET_RPCWALLET_H
 
+#include <memory>
 #include <string>
 
 class CRPCTable;

src/wallet/test/claim_rpc_tests.cpp

@@ -1,5 +1,3 @@
-#include <memory>
-#include <set>
 #include <utility>
 #include <vector>
 
@@ -13,7 +11,6 @@
 #include <test/test_bitcoin.h>
 #include <validation.h>
 #include <wallet/coincontrol.h>
-#include <wallet/test/wallet_test_fixture.h>
 #include <policy/policy.h>
 
 #include <boost/test/unit_test.hpp>
@@ -125,7 +122,7 @@ void rollbackBlock(const std::vector<uint256>& ids) {
     }
     // totally weird that invalidateblock is async
     while (GetMainSignals().CallbacksPending())
-        usleep(5000);
+        MilliSleep(5);
 }
 
 uint256 AbandonAClaim(const uint256& txid, bool isSupport = false) {

src/wallet/test/wallet_tests.cpp

@@ -24,6 +24,7 @@
 extern UniValue importmulti(const JSONRPCRequest& request);
 extern UniValue dumpwallet(const JSONRPCRequest& request);
 extern UniValue importwallet(const JSONRPCRequest& request);
+extern CChainState g_chainstate;
 
 BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup)
 
@@ -206,7 +207,7 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64
     CBlockIndex* block = nullptr;
     if (blockTime > 0) {
         LOCK(cs_main);
-        auto inserted = mapBlockIndex.emplace(GetRandHash(), new CBlockIndex);
+        auto inserted = g_chainstate.mapBlockIndex.emplace(GetRandHash(), new CBlockIndex);
         assert(inserted.second);
         const uint256& hash = inserted.first->first;
         block = inserted.first->second;