optimized hash calc, fixed bad nChainTx, re-used db connection

src/claimtrie/forks.cpp

@@ -254,14 +254,14 @@ uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, int ta
     childHashQuery++;
 
     std::vector<uint256> claimHashes;
-    //if (takeoverHeight > 0) {
+    if (takeoverHeight > 0) {
         COutPoint p;
         for (auto &&row: claimHashQuery << nNextHeight << name) {
             row >> p.hash >> p.n;
             claimHashes.push_back(getValueHash(p, takeoverHeight));
         }
         claimHashQuery++;
-    //}
+    }
 
     auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
     auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(claimHashes));

src/claimtrie/trie.cpp

@@ -40,10 +40,11 @@ static const sqlite::sqlite_config sharedConfig {
 void applyPragmas(sqlite::database& db, std::size_t cache)
 {
     db << "PRAGMA cache_size=-" + std::to_string(cache); // in -KB
-    db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit
-    db << "PRAGMA journal_mode=WAL";
     db << "PRAGMA temp_store=MEMORY";
     db << "PRAGMA case_sensitive_like=true";
+    db << "PRAGMA journal_mode=WAL";
+    db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit; we handle that elsewhere
+    db << "PRAGMA wal_autocheckpoint=4000"; // 4k page size * 4000 = 16MB
 
     db.define("POPS", [](std::string s) -> std::string { if (!s.empty()) s.pop_back(); return s; });
     db.define("REVERSE", [](std::vector<uint8_t> s) -> std::vector<uint8_t> { std::reverse(s.begin(), s.end()); return s; });
@@ -71,7 +72,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
                        nExtendedClaimExpirationForkHeight(nExtendedClaimExpirationForkHeight),
                        nAllClaimsInMerkleForkHeight(nAllClaimsInMerkleForkHeight)
 {
-    applyPragmas(db, 5U * 1024U); // in KB
+    applyPragmas(db, cacheBytes >> 10U); // in KB
 
     db << "CREATE TABLE IF NOT EXISTS node (name BLOB NOT NULL PRIMARY KEY, "
           "parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
@@ -134,10 +135,12 @@ bool CClaimTrie::SyncToDisk()
     return rc == SQLITE_OK;
 }
 
-bool CClaimTrie::empty()
+bool CClaimTrie::empty() // only used for testing
 {
+    sqlite::database local(dbFile, sharedConfig);
+    applyPragmas(local, 100);
     int64_t count;
-    db << "SELECT COUNT(*) FROM (SELECT 1 FROM claim WHERE activationHeight < ?1 AND expirationHeight >= ?1 LIMIT 1)" << nNextHeight >> count;
+    local << "SELECT COUNT(*) FROM (SELECT 1 FROM claim WHERE activationHeight < ?1 AND expirationHeight >= ?1 LIMIT 1)" << nNextHeight >> count;
     return count == 0;
 }
 
@@ -425,11 +428,13 @@ uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, int takeov
     };
     childHashQuery++;
 
+    if (takeoverHeight > 0) {
         CClaimValue claim;
         if (getInfoForName(name, claim)) {
             auto valueHash = getValueHash(claim.outPoint, takeoverHeight);
             vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
         }
+    }
 
     return vchToHash.empty() ? one : Hash(vchToHash.begin(), vchToHash.end());
 }
@@ -450,7 +455,7 @@ bool CClaimTrieCacheBase::checkConsistency()
     auto query = db << "SELECT n.name, n.hash, "
                         "IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
                         "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM node n "
-                        "WHERE n.name IN (SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 56789) OR LENGTH(n.parent) < 2";
+                        "WHERE n.name IN (SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 100000) OR LENGTH(n.parent) < 2";
     for (auto&& row: query) {
         std::string name;
         uint256 hash;
@@ -517,21 +522,21 @@ extern const std::string proofClaimQuery_s =
     "ORDER BY n.name";
 
 CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
-    : base(base), db(base->dbFile, sharedConfig), transacting(false),
+    : base(base), db(base->db.connection()), transacting(false),
       childHashQuery(db << childHashQuery_s),
       claimHashQuery(db << claimHashQuery_s),
       claimHashQueryLimit(db << claimHashQueryLimit_s)
 {
     assert(base);
     nNextHeight = base->nNextHeight;
-
-    applyPragmas(db, base->dbCacheBytes >> 10U); // in KB
 }
 
 void CClaimTrieCacheBase::ensureTransacting()
 {
     if (!transacting) {
         transacting = true;
+        int isNotInTransaction = sqlite3_get_autocommit(db.connection().get());
+        assert(isNotInTransaction);
         db << "BEGIN";
     }
 }
@@ -730,7 +735,16 @@ bool CClaimTrieCacheBase::incrementBlock()
           "UNION SELECT nodeName FROM support WHERE expirationHeight = ?1 OR activationHeight = ?1)"
           << nNextHeight;
 
-    auto insertTakeoverQuery = db << "INSERT INTO takeover(name, height, claimID) VALUES(?, ?, ?)";
+    insertTakeovers();
+
+    nNextHeight++;
+    return true;
+}
+
+void CClaimTrieCacheBase::insertTakeovers(bool allowReplace) {
+    auto insertTakeoverQuery = allowReplace ?
+            db << "INSERT OR REPLACE INTO takeover(name, height, claimID) VALUES(?, ?, ?)" :
+            db << "INSERT INTO takeover(name, height, claimID) VALUES(?, ?, ?)";
 
     // takeover handling:
     db << "SELECT name FROM node WHERE hash IS NULL"
@@ -768,9 +782,6 @@ bool CClaimTrieCacheBase::incrementBlock()
     };
 
     insertTakeoverQuery.used(true);
-
-    nNextHeight++;
-    return true;
 }
 
 bool CClaimTrieCacheBase::activateAllFor(const std::string& name)

src/claimtrie/trie.h

@@ -127,6 +127,7 @@ protected:
     bool deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims);
     void ensureTreeStructureIsUpToDate();
     void ensureTransacting();
+    void insertTakeovers(bool allowReplace=false);
 
 private:
     bool transacting;

src/test/claimtriecache_tests.cpp

@@ -15,7 +15,7 @@ class CClaimTrieCacheTest : public CClaimTrieCacheBase
 public:
     explicit CClaimTrieCacheTest(CClaimTrie* base): CClaimTrieCacheBase(base)
     {
-        nNextHeight = 2;
+        nNextHeight = std::max(2, nNextHeight);
     }
 
     bool insertClaimIntoTrie(const std::string& key, const CClaimValue& value)
@@ -63,6 +63,10 @@ public:
         assert(!ret || nodeName == key);
         return ret;
     }
+
+    void processTakeovers() {
+        insertTakeovers(true);
+    }
 };
 
 BOOST_FIXTURE_TEST_SUITE(claimtriecache_tests, RegTestingSetup)
@@ -97,110 +101,122 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     COutPoint tx6OutPoint(tx6.GetHash(), 0);
 
     uint256 hash1;
-    hash1.SetHex("71c7b8d35b9a3d7ad9a1272b68972979bbd18589f1efe6f27b0bf260a6ba78fa");
+    hash1.SetHex("917106c2e4a5d454c6463e366ec2c70fde57a3deacb36f566fc0c8568e4523d5");
 
     uint256 hash2;
-    hash2.SetHex("c4fc0e2ad56562a636a0a237a96a5f250ef53495c2cb5edd531f087a8de83722");
+    hash2.SetHex("79d0482510cb3da12b31096d459f8221cd094d268c03d96d897c932716e130e5");
 
     uint256 hash3;
-    hash3.SetHex("baf52472bd7da19fe1e35116cfb3bd180d8770ffbe3ae9243df1fb58a14b0975");
+    hash3.SetHex("14ba954095acc42e8a5c1c3c1ca1203a1f1fb44d66b4141272cbb5a16837e3e6");
 
     uint256 hash4;
-    hash4.SetHex("c73232a755bf015f22eaa611b283ff38100f2a23fb6222e86eca363452ba0c51");
+    hash4.SetHex("446b9dd04a1ff062c984cbbf0d5ab5aaea4ee3db1a9b65c5d06be0cb2090dbcf");
 
+    auto dataDir = GetDataDir() / "merkle_test";
+    fs::create_directories(dataDir);
+    CClaimTrie trie(10*1024*1024, true, 3, dataDir.string(), 1000, 1000, -1, 1000, 1000, 1000, 1000, 1);
     {
-        CClaimTrieCacheTest ntState(pclaimTrie);
+        CClaimTrieCacheTest ntState(&trie);
-        ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, {}, 50, 0, 0));
+        ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, {}, 50, 2, 2));
-        ntState.insertClaimIntoTrie(std::string("test2"), CClaimValue(tx2OutPoint, {}, 50, 0, 0));
+        ntState.insertClaimIntoTrie(std::string("test2"), CClaimValue(tx2OutPoint, {}, 50, 2, 2));
 
-        BOOST_CHECK(pclaimTrie->empty());
+        BOOST_CHECK(trie.empty());
         BOOST_CHECK_EQUAL(ntState.getTotalClaimsInTrie(), 2U);
+        ntState.processTakeovers();
         BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash1);
 
-        ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, {}, 50, 1, 1));
+        ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, {}, 50, 3, 3));
+        ntState.processTakeovers();
         BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash1);
-        ntState.insertClaimIntoTrie(std::string("tes"), CClaimValue(tx4OutPoint, {}, 50, 0, 0));
+        ntState.insertClaimIntoTrie(std::string("tes"), CClaimValue(tx4OutPoint, {}, 50, 2, 2));
+        ntState.processTakeovers();
         BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
         ntState.insertClaimIntoTrie(std::string("testtesttesttest"),
-                                    CClaimValue(tx5OutPoint, {}, 50, 0, 0));
+                                    CClaimValue(tx5OutPoint, {}, 50, 2, 2));
         ntState.removeClaimFromTrie(std::string("testtesttesttest"), tx5OutPoint);
+        ntState.processTakeovers();
         BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
         ntState.flush();
 
-        BOOST_CHECK(!pclaimTrie->empty());
+        BOOST_CHECK(!trie.empty());
         BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
         BOOST_CHECK(ntState.checkConsistency());
     }
     {
-        CClaimTrieCacheTest ntState1(pclaimTrie);
+        CClaimTrieCacheTest ntState1(&trie);
         ntState1.removeClaimFromTrie(std::string("test"), tx1OutPoint);
         ntState1.removeClaimFromTrie(std::string("test2"), tx2OutPoint);
         ntState1.removeClaimFromTrie(std::string("test"), tx3OutPoint);
         ntState1.removeClaimFromTrie(std::string("tes"), tx4OutPoint);
 
+        ntState1.processTakeovers();
         BOOST_CHECK_EQUAL(ntState1.getMerkleHash(), hash0);
     }
     {
-        CClaimTrieCacheTest ntState2(pclaimTrie);
+        CClaimTrieCacheTest ntState2(&trie);
-        ntState2.insertClaimIntoTrie(std::string("abab"), CClaimValue(tx6OutPoint, {}, 50, 0, 200));
+        ntState2.insertClaimIntoTrie(std::string("abab"), CClaimValue(tx6OutPoint, {}, 50, 2, 200));
         ntState2.removeClaimFromTrie(std::string("test"), tx1OutPoint);
 
+        ntState2.processTakeovers();
         BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3);
 
         ntState2.flush();
 
-        BOOST_CHECK(!pclaimTrie->empty());
+        BOOST_CHECK(!trie.empty());
         BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3);
         BOOST_CHECK(ntState2.checkConsistency());
     }
     {
-        CClaimTrieCacheTest ntState3(pclaimTrie);
+        CClaimTrieCacheTest ntState3(&trie);
-        ntState3.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, {}, 50, 0, 0));
+        ntState3.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, {}, 50, 2, 2));
+        ntState3.processTakeovers();
         BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4);
         ntState3.flush();
-        BOOST_CHECK(!pclaimTrie->empty());
+        BOOST_CHECK(!trie.empty());
         BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4);
         BOOST_CHECK(ntState3.checkConsistency());
     }
     {
-        CClaimTrieCacheTest ntState4(pclaimTrie);
+        CClaimTrieCacheTest ntState4(&trie);
         ntState4.removeClaimFromTrie(std::string("abab"), tx6OutPoint);
+        ntState4.processTakeovers();
         BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
         ntState4.flush();
-        BOOST_CHECK(!pclaimTrie->empty());
+        BOOST_CHECK(!trie.empty());
         BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
         BOOST_CHECK(ntState4.checkConsistency());
     }
     {
-        CClaimTrieCacheTest ntState5(pclaimTrie);
+        CClaimTrieCacheTest ntState5(&trie);
         ntState5.removeClaimFromTrie(std::string("test"), tx3OutPoint);
-
+        ntState5.processTakeovers();
         BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
         ntState5.flush();
-        BOOST_CHECK(!pclaimTrie->empty());
+        BOOST_CHECK(!trie.empty());
         BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
         BOOST_CHECK(ntState5.checkConsistency());
     }
     {
-        CClaimTrieCacheTest ntState6(pclaimTrie);
+        CClaimTrieCacheTest ntState6(&trie);
-        ntState6.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, {}, 50, 1, 1));
+        ntState6.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, {}, 50, 3, 3));
-
+        ntState6.processTakeovers();
         BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
         ntState6.flush();
-        BOOST_CHECK(!pclaimTrie->empty());
+        BOOST_CHECK(!trie.empty());
         BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
         BOOST_CHECK(ntState6.checkConsistency());
     }
     {
-        CClaimTrieCacheTest ntState7(pclaimTrie);
+        CClaimTrieCacheTest ntState7(&trie);
         ntState7.removeClaimFromTrie(std::string("test"), tx3OutPoint);
         ntState7.removeClaimFromTrie(std::string("test"), tx1OutPoint);
         ntState7.removeClaimFromTrie(std::string("tes"), tx4OutPoint);
         ntState7.removeClaimFromTrie(std::string("test2"), tx2OutPoint);
 
+        ntState7.processTakeovers();
         BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
         ntState7.flush();
-        BOOST_CHECK(pclaimTrie->empty());
+        BOOST_CHECK(trie.empty());
         BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
         BOOST_CHECK(ntState7.checkConsistency());
     }

src/txdb.cpp

@@ -30,10 +30,11 @@ CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe)
     : db(fMemory ? ":memory:" : (GetDataDir() / "coins.sqlite").string(), sharedConfig)
 {
     db << "PRAGMA cache_size=-" + std::to_string(nCacheSize >> 10); // in -KB
-    db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit
-    db << "PRAGMA journal_mode=WAL";
     db << "PRAGMA temp_store=MEMORY";
     db << "PRAGMA case_sensitive_like=true";
+    db << "PRAGMA journal_mode=WAL";
+    db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit; we handle that elsewhere
+    db << "PRAGMA wal_autocheckpoint=4000"; // 4k page size * 4000 = 16MB
 
     db << "CREATE TABLE IF NOT EXISTS unspent (txID BLOB NOT NULL COLLATE BINARY, txN INTEGER NOT NULL, "
           "isCoinbase INTEGER NOT NULL, blockHeight INTEGER NOT NULL, amount INTEGER NOT NULL, "
@@ -174,10 +175,11 @@ CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe)
     : db(fMemory ? ":memory:" : (GetDataDir() / "block_index.sqlite").string(), sharedConfig)
 {
     db << "PRAGMA cache_size=-" + std::to_string(nCacheSize >> 10); // in -KB
-    db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit
-    db << "PRAGMA journal_mode=WAL";
     db << "PRAGMA temp_store=MEMORY";
     db << "PRAGMA case_sensitive_like=true";
+    db << "PRAGMA journal_mode=WAL";
+    db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit; we handle that elsewhere
+    db << "PRAGMA wal_autocheckpoint=4000"; // 4k page size * 4000 = 16MB
 
     db << "CREATE TABLE IF NOT EXISTS block_file ("
           "file INTEGER NOT NULL PRIMARY KEY, "
@@ -383,13 +385,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams,
             >> pindexNew->nNonce
             >> pindexNew->nStatus;
 
-        if ((pindexNew->nHeight & 0x3ff) == 0x3ff) { // don't check for shutdown on every single block
+        // nChainTx gets set later; don't set it here or you'll mess up the Unlinked list
-            boost::this_thread::interruption_point();
-            if (ShutdownRequested())
-                return false;
-        }
-
-        pindexNew->nChainTx = pindexNew->pprev ? pindexNew->pprev->nChainTx + pindexNew->nTx : pindexNew->nTx;
 
         if (!CheckProofOfWork(pindexNew->GetBlockPoWHash(), pindexNew->nBits, consensusParams))
         {
@@ -397,6 +393,12 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams,
             LogPrintf("%s: CheckProofOfWorkFailed: %s (hash %s, nBits=%x, nTime=%d)\n", __func__, pindexNew->GetBlockPoWHash().GetHex(), pindexNew->GetBlockHash().GetHex(), pindexNew->nBits, pindexNew->nTime);
             return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
         }
+
+        if ((pindexNew->nHeight & 0x3ff) == 0x3ff) { // don't check for shutdown on every single block
+            boost::this_thread::interruption_point();
+            if (ShutdownRequested())
+                return false;
+        }
     }
 
     return true;

src/validation.cpp

@@ -2531,7 +2531,7 @@ CBlockIndex* CChainState::FindMostWorkChain() {
 
         // Find the best candidate header.
         {
-            std::set<CBlockIndex*, CBlockIndexWorkComparator>::reverse_iterator it = setBlockIndexCandidates.rbegin();
+            auto it = setBlockIndexCandidates.rbegin();
             if (it == setBlockIndexCandidates.rend())
                 return nullptr;
             pindexNew = *it;