replaced leveldb with sqlite for claimtrie db

tweaks

src/Makefile.am

@@ -131,6 +131,7 @@ BITCOIN_CORE_H = \
   keystore.h \
   lbry.h \
   dbwrapper.h \
+  sqlwrapper.h \
   limitedmap.h \
   logging.h \
   memusage.h \
@@ -479,7 +480,7 @@ lbrycrdd_LDADD = \
   $(LIBMEMENV) \
   $(LIBSECP256K1)
 
-lbrycrdd_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
+lbrycrdd_LDADD += -lsqlite3 $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) -lsqlite3
 
 # lbrycrd-cli binary #
 lbrycrd_cli_SOURCES = bitcoin-cli.cpp

src/Makefile.bench.include

@@ -34,7 +34,7 @@ nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES)
 
 bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) $(BOOST_CPPFLAGS) -I$(builddir)/bench/
 bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
-bench_bench_bitcoin_LDADD = \
+bench_bench_bitcoin_LDADD = -lsqlite3 \
   $(LIBBITCOIN_WALLET) \
   $(LIBBITCOIN_SERVER) \
   $(LIBBITCOIN_COMMON) \
@@ -55,7 +55,7 @@ if ENABLE_WALLET
 bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp
 endif
 
-bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
+bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -lsqlite3
 bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
 
 CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES)

src/Makefile.test.include

@@ -121,11 +121,11 @@ if ENABLE_WALLET
 test_test_lbrycrd_LDADD += $(LIBBITCOIN_WALLET)
 endif
 
-test_test_lbrycrd_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
+test_test_lbrycrd_LDADD += -lsqlite3 $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
   $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
 test_test_lbrycrd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
-test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS)
+test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) -lsqlite3
 test_test_lbrycrd_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static
 
 if ENABLE_ZMQ

src/claimtrie.cpp

@@ -8,8 +8,6 @@
 #include <algorithm>
 #include <memory>
 
-#include <boost/scoped_ptr.hpp>
-
 static const uint256 one = uint256S("0000000000000000000000000000000000000000000000000000000000000001");
 
 std::vector<unsigned char> heightToVch(int n)
@@ -123,13 +121,13 @@ void CClaimTrieData::reorderClaims(const supportEntryType& supports)
                 claim.nEffectiveAmount += support.nAmount;
     }
 
-    std::make_heap(claims.begin(), claims.end());
+    std::sort(claims.rbegin(), claims.rend());
 }
 
 CClaimTrie::CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor)
 {
     nProportionalDelayFactor = proportionalDelayFactor;
-    db.reset(new CDBWrapper(GetDataDir() / "claimtrie", 100 * 1024 * 1024, fMemory, fWipe, false));
+    db.reset(new CSqlWrapper(GetDataDir() / "claimtrie.sqlite3.db", 200 * 1024 * 1024, fMemory, fWipe));
 }
 
 bool CClaimTrie::SyncToDisk()
@@ -138,7 +136,7 @@ bool CClaimTrie::SyncToDisk()
 }
 
 template <typename Key, typename Container>
-typename Container::value_type* getQueue(CDBWrapper& db, uint8_t dbkey, const Key& key, Container& queue, bool create)
+typename Container::value_type* getQueue(CSqlWrapper& db, uint8_t dbkey, const Key& key, Container& queue, bool create)
 {
     auto itQueue = queue.find(key);
     if (itQueue != queue.end())
@@ -200,7 +198,7 @@ typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow(co
 template <>
 typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueueCacheRow<CClaimValue>(int nHeight, bool createIfNotExists)
 {
-    return getQueue(*(base->db), EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists);
+    return getQueue(*(base->db), CLAIM_EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists);
 }
 
 template <>
@@ -218,8 +216,14 @@ typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueu
 
 bool CClaimTrieCacheBase::haveClaim(const std::string& name, const COutPoint& outPoint) const
 {
-    auto it = find(name);
+    auto it = nodesToAddOrUpdate.find(name);
-    return it && it->haveClaim(outPoint);
+    if (it && it->haveClaim(outPoint))
+        return true;
+    if (it || nodesToDelete.count(name))
+        return false;
+    CClaimTrieDataNode node;
+    node.childrenSerialization = false;
+    return base->find(name, node) && node.data.haveClaim(outPoint);
 }
 
 bool CClaimTrieCacheBase::haveSupport(const std::string& name, const COutPoint& outPoint) const
@@ -272,39 +276,63 @@ bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COut
     return haveInQueue<CSupportValue>(name, outPoint, nValidAtHeight);
 }
 
-std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const
+void CClaimTrie::recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<void(const std::string&, const CClaimTrieDataNode&)> function) const {
+    function(name, current);
+    for (auto& child: current.children) {
+        CClaimTrieDataNode node;
+        if (find(child.second, node))
+            recurseAllHashedNodes(name + child.first, node, function);
+    }
+}
+
+std::size_t CClaimTrie::getTotalNamesInTrie() const
 {
     std::size_t count = 0;
-    for (auto it = base->cbegin(); it != base->cend(); ++it)
+    CClaimTrieDataNode node;
-        if (!it->empty()) ++count;
+    if (find("", node))
+        recurseAllHashedNodes("", node, [&count](const std::string&, const CClaimTrieDataNode& node) {
+            count += !node.data.empty();
+        });
     return count;
 }
 
-std::size_t CClaimTrieCacheBase::getTotalClaimsInTrie() const
+std::size_t CClaimTrie::getTotalClaimsInTrie() const
 {
     std::size_t count = 0;
-    for (auto it = base->cbegin(); it != base->cend(); ++it)
+    CClaimTrieDataNode node;
-        count += it->claims.size();
+    if (find("", node))
+        recurseAllHashedNodes("", node, [&count](const std::string&, const CClaimTrieDataNode& node) {
+            count += node.data.claims.size();
+        });
     return count;
 }
 
-CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) const
+CAmount CClaimTrie::getTotalValueOfClaimsInTrie(bool fControllingOnly) const
 {
     CAmount value_in_subtrie = 0;
-    for (auto it = base->cbegin(); it != base->cend(); ++it) {
+    std::size_t count = 0;
-        for (const auto& claim : it->claims) {
+    CClaimTrieDataNode node;
-            value_in_subtrie += claim.nAmount;
+    if (find("", node))
-            if (fControllingOnly)
+        recurseAllHashedNodes("", node, [&value_in_subtrie, fControllingOnly](const std::string&, const CClaimTrieDataNode& node) {
-                break;
+            for (const auto& claim : node.data.claims) {
-        }
+                value_in_subtrie += claim.nAmount;
-    }
+                if (fControllingOnly)
+                    break;
+            }
+        });
     return value_in_subtrie;
 }
 
 bool CClaimTrieCacheBase::getInfoForName(const std::string& name, CClaimValue& claim) const
 {
-    auto it = find(name);
+    auto it = nodesToAddOrUpdate.find(name);
-    return it && it->getBestClaim(claim);
+    if (it && it->getBestClaim(claim))
+        return true;
+    if (it || nodesToDelete.count(name))
+        return false;
+    CClaimTrieDataNode node;
+    node.childrenSerialization = false;
+    return base->find(name, node) && node.data.getBestClaim(claim);
 }
 
 CClaimsForNameType CClaimTrieCacheBase::getClaimsForName(const std::string& name) const
@@ -313,10 +341,16 @@ CClaimsForNameType CClaimTrieCacheBase::getClaimsForName(const std::string& name
     int nLastTakeoverHeight = 0;
     auto supports = getSupportsForName(name);
 
-    if (auto it = find(name)) {
+    CClaimTrieDataNode node;
+    node.childrenSerialization = false;
+    if (auto it = nodesToAddOrUpdate.find(name)) {
         claims = it->claims;
         nLastTakeoverHeight = it->nHeightOfLastTakeover;
     }
+    else if (!nodesToDelete.count(name) && base->find(name, node)) {
+        claims = node.data.claims;
+        nLastTakeoverHeight = node.data.nHeightOfLastTakeover;
+    }
     return {std::move(claims), std::move(supports), nLastTakeoverHeight, name};
 }
 
@@ -381,60 +415,97 @@ uint256 recursiveMerkleHash(TIterator& it, const iCbType<TIterator>& process, co
     return Hash(vchToHash.begin(), vchToHash.end());
 }
 
-bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed)
+bool CClaimTrie::checkConsistency(const uint256& rootHash) const
 {
-    struct CRecursiveBreak : public std::exception {};
+    CClaimTrieDataNode node;
+    if (!find("", node) || node.data.hash != rootHash) {
+        if (rootHash == one)
+            return true;
 
-    using iterator = CClaimTrie::const_iterator;
+        return error("Mismatched root claim trie hashes. This may happen when there is not a clean process shutdown. Please run with -reindex.");
-    iCbType<iterator> verify = [&failed](iterator& it) {
-        if (!it.hasChildren()) {
-            // we don't allow a situation of no children and no claims; no empty leaf nodes allowed
-            failed = it.key();
-            throw CRecursiveBreak();
-        }
-    };
-
-    iCbType<iterator> process = [&failed, &process, &verify](iterator& it) {
-        if (it->hash != recursiveMerkleHash(it, process, verify)) {
-            failed = it.key();
-            throw CRecursiveBreak();
-        }
-    };
-
-    try {
-        process(it);
-    } catch (const CRecursiveBreak&) {
-        return false;
     }
-    return true;
+
+    bool success = true;
+    recurseAllHashedNodes("", node, [&success, this](const std::string& name, const CClaimTrieDataNode& node) {
+        if (!success) return;
+
+        success &= contains(name);
+
+        std::vector<uint8_t> vchToHash;
+        const auto pos = name.size();
+        for (auto& child : node.children) {
+            auto key = name + child.first;
+            auto hash = child.second;
+            completeHash(hash, key, pos);
+            vchToHash.push_back(key[pos]);
+            vchToHash.insert(vchToHash.end(), hash.begin(), hash.end());
+        }
+
+        CClaimValue claim;
+        if (node.data.getBestClaim(claim)) {
+            uint256 valueHash = getValueHash(claim.outPoint, node.data.nHeightOfLastTakeover);
+            vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
+        } else {
+            success &= !node.children.empty(); // we disallow leaf nodes without claims
+        }
+
+        success &= node.data.hash == Hash(vchToHash.begin(), vchToHash.end());
+    });
+
+    return success;
 }
 
-bool CClaimTrieCacheBase::checkConsistency() const
+std::vector<std::pair<std::string, CClaimTrieDataNode>> CClaimTrie::nodes(const std::string &key) const {
-{
+    std::vector<std::pair<std::string, CClaimTrieDataNode>> ret;
-    if (base->empty())
+    CClaimTrieDataNode node;
-        return true;
 
-    auto it = base->cbegin();
+    if (!find("", node))
-    std::string failed;
+        return ret;
-    auto consistent = recursiveCheckConsistency(it, failed);
+    ret.emplace_back("", node);
-    if (!consistent) {
+
-        LogPrintf("\nPrinting base tree from its parent:\n");
+    std::string partialKey = key;
-        auto basePath = base->nodes(failed);
+
-        if (basePath.size() > 1) basePath.pop_back();
+    while (!node.children.empty()) {
-        dumpToLog(basePath.back(), false);
+        // auto it = node.children.lower_bound(partialKey); // for using a std::map
-        auto cachePath = nodesToAddOrUpdate.nodes(failed);
+        auto it = std::lower_bound(node.children.begin(), node.children.end(), std::make_pair(partialKey, uint256()));
-        if (!cachePath.empty()) {
+        if (it != node.children.end() && it->first == partialKey) {
-            LogPrintf("\nPrinting %s's parent from cache:\n", failed);
+            // we're completely done
-            if (cachePath.size() > 1) cachePath.pop_back();
+            if (find(it->second, node))
-            dumpToLog(cachePath.back(), false);
+                ret.emplace_back(key, node);
-        }
+            break;
-        if (!nodesToDelete.empty()) {
-            std::string joined;
-            for (const auto &piece : nodesToDelete) joined += ", " + piece;
-            LogPrintf("Nodes to be deleted: %s\n", joined.substr(2));
         }
+        if (it != node.children.begin()) --it;
+        const auto count = match(partialKey, it->first);
+
+        if (count != it->first.size()) break;
+        if (count == partialKey.size()) break;
+        partialKey = partialKey.substr(count);
+        if (find(it->second, node))
+            ret.emplace_back(key.substr(0, key.size() - partialKey.size()), node);
+        else break;
     }
-    return consistent;
+
+    return ret;
+}
+
+bool CClaimTrie::contains(const std::string &key) const {
+    return db->Exists(std::make_pair(TRIE_NODE_BY_NAME, key));
+}
+
+bool CClaimTrie::empty() const {
+    return !contains("");
+}
+
+bool CClaimTrie::find(const std::string &key, CClaimTrieDataNode &node) const {
+    uint256 hash;
+    if (!db->Read(std::make_pair(TRIE_NODE_BY_NAME, key), hash))
+        return false;
+    auto found = find(hash, node);
+    return found;
+}
+
+bool CClaimTrie::find(const uint256 &key, CClaimTrieDataNode &node) const {
+    return db->Read(std::make_pair(TRIE_NODE_BY_HASH, key), node);
 }
 
 bool CClaimTrieCacheBase::getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const
@@ -451,7 +522,7 @@ bool CClaimTrieCacheBase::getClaimById(const uint160& claimId, std::string& name
 }
 
 template <typename K, typename T>
-void BatchWrite(CDBBatch& batch, uint8_t dbkey, const K& key, const std::vector<T>& value)
+void BatchWrite(CSqlBatch& batch, uint8_t dbkey, const K& key, const std::vector<T>& value)
 {
     if (value.empty()) {
         batch.Erase(std::make_pair(dbkey, key));
@@ -461,7 +532,7 @@ void BatchWrite(CDBBatch& batch, uint8_t dbkey, const K& key, const std::vector<
 }
 
 template <typename Container>
-void BatchWriteQueue(CDBBatch& batch, uint8_t dbkey, const Container& queue)
+void BatchWriteQueue(CSqlBatch& batch, uint8_t dbkey, const Container& queue)
 {
     for (auto& itQueue : queue)
         BatchWrite(batch, dbkey, itQueue.first, itQueue.second);
@@ -469,7 +540,7 @@ void BatchWriteQueue(CDBBatch& batch, uint8_t dbkey, const Container& queue)
 
 bool CClaimTrieCacheBase::flush()
 {
-    CDBBatch batch(*(base->db));
+    CSqlBatch batch(*(base->db));
 
     for (const auto& claim : claimsToDeleteFromByIdIndex) {
         auto it = std::find_if(claimsToAddToByIdIndex.begin(), claimsToAddToByIdIndex.end(),
@@ -486,99 +557,79 @@ bool CClaimTrieCacheBase::flush()
 
     getMerkleHash();
 
+    std::set<std::string> forDeletion;
     for (const auto& nodeName : nodesToDelete) {
-        if (nodesToAddOrUpdate.contains(nodeName))
+        // TODO: we don't need to deserialize all the nodes right here
-            continue;
+        // we could be smarter about this and fill in the whole list in removeClaimFromTrie
         auto nodes = base->nodes(nodeName);
-        base->erase(nodeName);
         for (auto& node : nodes)
-            if (!node)
+            forDeletion.insert(node.first);
-                batch.Erase(std::make_pair(TRIE_NODE, node.key()));
     }
 
     for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) {
-        auto old = base->find(it.key());
+        forDeletion.erase(it.key());
-        if (!old || old.data() != it.data()) {
+        if (!dirtyNodes.count(it.key()))
-            base->copy(it);
+            continue;
-            batch.Write(std::make_pair(TRIE_NODE, it.key()), it.data());
+
-        }
+        CClaimTrieDataNode node;
+        node.data = it.data();
+        for (auto &child: it.children()) // ordering here is important
+            node.children.emplace_back(child.key().substr(it.key().size()), child->hash);
+
+        batch.Write(std::make_pair(TRIE_NODE_BY_HASH, it->hash), node);
+        batch.Write(std::make_pair(TRIE_NODE_BY_NAME, it.key()), it->hash);
+    }
+
+    for (auto& name: forDeletion) {
+        batch.Erase(std::make_pair(TRIE_NODE_BY_NAME, name));
     }
 
     BatchWriteQueue(batch, SUPPORT, supportCache);
+
     BatchWriteQueue(batch, CLAIM_QUEUE_ROW, claimQueueCache);
     BatchWriteQueue(batch, CLAIM_QUEUE_NAME_ROW, claimQueueNameCache);
-    BatchWriteQueue(batch, EXP_QUEUE_ROW, expirationQueueCache);
+    BatchWriteQueue(batch, CLAIM_EXP_QUEUE_ROW, expirationQueueCache);
+
     BatchWriteQueue(batch, SUPPORT_QUEUE_ROW, supportQueueCache);
     BatchWriteQueue(batch, SUPPORT_QUEUE_NAME_ROW, supportQueueNameCache);
     BatchWriteQueue(batch, SUPPORT_EXP_QUEUE_ROW, supportExpirationQueueCache);
 
     base->nNextHeight = nNextHeight;
-    if (!nodesToAddOrUpdate.empty())
+    if (!nodesToAddOrUpdate.empty() && (LogAcceptCategory(BCLog::CLAIMS) || LogAcceptCategory(BCLog::BENCH))) {
-        LogPrint(BCLog::CLAIMS, "Cache size: %zu from base size: %zu on block %d\n", nodesToAddOrUpdate.height(), base->height(), nNextHeight);
+        LogPrintf("TrieCache size: %zu nodes on block %d.\n",
+                nodesToAddOrUpdate.height(), nNextHeight);
+    }
     auto ret = base->db->WriteBatch(batch);
     clear();
     return ret;
 }
 
-bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
+bool CClaimTrieCacheBase::validateTrieConsistency(const CBlockIndex* tip)
 {
-    LogPrintf("Loading the claim trie from disk...\n");
+    if (!tip || tip->nHeight < 1)
-
+        return true;
-    base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0;
-
-    clear();
-    base->clear();
-    boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
-
-    std::vector<std::pair<std::string, uint256>> hashesOnEmptyNodes;
-
-    for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
-        std::pair<uint8_t, std::string> key;
-        if (!pcursor->GetKey(key) || key.first != TRIE_NODE)
-            continue;
-
-        CClaimTrieData data;
-        if (pcursor->GetValue(data)) {
-            if (data.empty()) {
-                // we have a situation where our old trie had many empty nodes
-                // we don't want to automatically throw those all into our prefix trie
-                hashesOnEmptyNodes.emplace_back(key.second, data.hash);
-                continue;
-            }
-
-            // nEffectiveAmount isn't serialized but it needs to be initialized (as done in reorderClaims):
-            auto supports = getSupportsForName(key.second);
-            data.reorderClaims(supports);
-            base->insert(key.second, std::move(data));
-        } else {
-            return error("%s(): error reading claim trie from disk", __func__);
-        }
-    }
-
-    CDBBatch batch(*(base->db));
-    for (auto& kvp: hashesOnEmptyNodes) {
-        auto hit = base->find(kvp.first);
-        if (hit != base->end())
-            hit->hash = kvp.second;
-        else {
-            // the first time the prefix trie is ran there will be many unused nodes
-            // we need to clean those out so that we can go faster next time
-            batch.Erase(std::make_pair(TRIE_NODE, kvp.first));
-        }
-    }
 
     LogPrintf("Checking claim trie consistency... ");
-    if (checkConsistency()) {
+    if (base->checkConsistency(tip->hashClaimTrie)) {
         LogPrintf("consistent\n");
-        if (tip && tip->hashClaimTrie != getMerkleHash())
-            return error("%s(): hashes don't match when reading claimtrie from disk", __func__);
-        base->db->WriteBatch(batch);
         return true;
     }
     LogPrintf("inconsistent!\n");
     return false;
 }
 
+bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
+{
+    base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0;
+    clear();
+
+    if (tip && (base->db->Exists(std::make_pair(TRIE_NODE, std::string())) || !base->db->Exists(std::make_pair(TRIE_NODE_BY_HASH, tip->hashClaimTrie)))) {
+        LogPrintf("The claim trie database contains deprecated data and will need to be rebuilt");
+        return false;
+    }
+    return validateTrieConsistency(tip);
+}
+
 CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) : base(base)
 {
     assert(base);
@@ -590,9 +641,9 @@ int CClaimTrieCacheBase::expirationTime() const
     return Params().GetConsensus().nOriginalClaimExpirationTime;
 }
 
-uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimTrie::iterator& it)
+uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it)
 {
-    using iterator = CClaimTrie::iterator;
+    using iterator = CClaimPrefixTrie::iterator;
     iCbType<iterator> process = [&process](iterator& it) {
         if (it->hash.IsNull())
             it->hash = recursiveMerkleHash(it, process);
@@ -604,54 +655,52 @@ uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimTrie::iterator& it
 uint256 CClaimTrieCacheBase::getMerkleHash()
 {
     auto it = nodesToAddOrUpdate.begin();
-    if (nodesToAddOrUpdate.empty() && nodesToDelete.empty())
+    if (it)
-        it = base->begin();
+        return recursiveComputeMerkleHash(it);
-    return !it ? one : recursiveComputeMerkleHash(it);
+    if (nodesToDelete.empty() && nodesAlreadyCached.empty()) {
+        CClaimTrieDataNode node;
+        node.childrenSerialization = false;
+        if (base->find("", node))
+            return node.data.hash; // it may be valuable to have base cache its current root hash
+    }
+    return one; // we have no data or we deleted everything
 }
 
-CClaimTrie::const_iterator CClaimTrieCacheBase::begin() const
+CClaimPrefixTrie::const_iterator CClaimTrieCacheBase::begin() const
 {
-    return nodesToAddOrUpdate.empty() && nodesToDelete.empty() ? base->cbegin() : nodesToAddOrUpdate.begin();
+    return nodesToAddOrUpdate.begin();
 }
 
-CClaimTrie::const_iterator CClaimTrieCacheBase::end() const
+CClaimPrefixTrie::const_iterator CClaimTrieCacheBase::end() const
 {
-    return nodesToAddOrUpdate.empty() && nodesToDelete.empty() ? base->cend() : nodesToAddOrUpdate.end();
+    return nodesToAddOrUpdate.end();
-}
-
-CClaimTrie::const_iterator CClaimTrieCacheBase::find(const std::string& name) const
-{
-    if (auto it = nodesToAddOrUpdate.find(name))
-        return it;
-    return base->find(name);
 }
 
 bool CClaimTrieCacheBase::empty() const
 {
-    return base->empty() && nodesToAddOrUpdate.empty();
+    return nodesToAddOrUpdate.empty();
 }
 
-CClaimTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& name, bool create)
+CClaimPrefixTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& name, bool create)
 {
-    // get data from the cache. if no data, create empty one
-    const auto insert = [this](CClaimTrie::iterator& it) {
-        auto& key = it.key();
-        // we only ever cache nodes once per cache instance
-        if (!nodesAlreadyCached.count(key)) {
-            // do not insert nodes that are already present
-            nodesAlreadyCached.insert(key);
-            nodesToAddOrUpdate.insert(key, it.data());
-        }
-    };
-
     // we need all parent nodes and their one level deep children
     // to calculate merkle hash
     auto nodes = base->nodes(name);
     for (auto& node: nodes) {
-        for (auto& child : node.children())
+        if (nodesAlreadyCached.insert(node.first).second) {
-            if (!nodesAlreadyCached.count(child.key()))
+            // do not insert nodes that are already present
-                nodesToAddOrUpdate.copy(child);
+            nodesToAddOrUpdate.insert(node.first, node.second.data);
-        insert(node);
+        }
+        for (auto& child : node.second.children) {
+            auto childKey = node.first + child.first;
+            if (nodesAlreadyCached.insert(childKey).second) {
+                CClaimTrieDataNode childNode;
+                childNode.childrenSerialization = false;
+                if (base->find(child.second, childNode)) {
+                    nodesToAddOrUpdate.insert(childKey, childNode.data);
+                }
+            }
+        }
     }
 
     auto it = nodesToAddOrUpdate.find(name);
@@ -677,10 +726,12 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16
         std::tie(claimId, takeoverHeight) = cit->second;
         return true;
     }
-    if (auto it = base->find(name)) {
+    CClaimTrieDataNode data;
-        takeoverHeight = it->nHeightOfLastTakeover;
+    data.childrenSerialization = false;
+    if (base->find(name, data)) {
+        takeoverHeight = data.data.nHeightOfLastTakeover;
         CClaimValue claim;
-        if (it->getBestClaim(claim)) {
+        if (data.data.getBestClaim(claim)) {
             claimId = claim.claimId;
             return true;
         }
@@ -690,8 +741,10 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16
 
 void CClaimTrieCacheBase::markAsDirty(const std::string& name, bool fCheckTakeover)
 {
-    for (auto& node : nodesToAddOrUpdate.nodes(name))
+    for (auto& node : nodesToAddOrUpdate.nodes(name)) {
+        dirtyNodes.insert(node.key());
         node->hash.SetNull();
+    }
 
     if (fCheckTakeover)
         namesToCheckForTakeover.insert(name);
@@ -712,7 +765,7 @@ bool CClaimTrieCacheBase::removeClaimFromTrie(const std::string& name, const COu
     auto it = cacheData(name, false);
 
     if (!it || !it->removeClaim(outPoint, claim)) {
-        LogPrint(BCLog::CLAIMS, "%s: Removing a claim was unsuccessful. name = %s, txhash = %s, nOut = %d", __func__, name, outPoint.hash.GetHex(), outPoint.n);
+        LogPrint(BCLog::CLAIMS, "%s: Removing a claim was unsuccessful. name = %s, txhash = %s, nOut = %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n);
         return false;
     }
 
@@ -963,11 +1016,14 @@ bool CClaimTrieCacheBase::removeSupportFromMap(const std::string& name, const CO
     return false;
 }
 
-void CClaimTrieCacheBase::dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase) const
+void CClaimTrieCacheBase::dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase) const
 {
+    if (!it) return;
+
     if (diffFromBase) {
-        auto hit = base->find(it.key());
+        CClaimTrieDataNode node;
-        if (hit && hit->hash == it->hash)
+        node.childrenSerialization = false;
+        if (base->find(it.key(), node) && node.data.hash == it->hash)
             return;
     }
 
@@ -1281,8 +1337,16 @@ int CClaimTrieCacheBase::getNumBlocksOfContinuousOwnership(const std::string& na
         that->removalWorkaround.erase(hit);
         return 0;
     }
-    auto it = find(name);
+    auto it = nodesToAddOrUpdate.find(name);
-    return it && !it->empty() ? nNextHeight - it->nHeightOfLastTakeover : 0;
+    if (it && !it->empty())
+        return nNextHeight - it->nHeightOfLastTakeover;
+    if (it) // we specifically ignore deleted nodes here to allow this to fall into the base lookup in that scenario
+        return 0;
+    CClaimTrieDataNode node;
+    node.childrenSerialization = false;
+    if (base->find(name, node) && !node.data.empty())
+        return nNextHeight - node.data.nHeightOfLastTakeover;
+    return 0;
 }
 
 int CClaimTrieCacheBase::getDelayForName(const std::string& name) const
@@ -1311,6 +1375,7 @@ bool CClaimTrieCacheBase::clear()
 {
     nodesToAddOrUpdate.clear();
     claimsToAddToByIdIndex.clear();
+    dirtyNodes.clear();
     supportCache.clear();
     nodesToDelete.clear();
     claimsToDeleteFromByIdIndex.clear();

src/claimtrie.h

@@ -4,7 +4,7 @@
 #include <amount.h>
 #include <chain.h>
 #include <chainparams.h>
-#include <dbwrapper.h>
+#include <sqlwrapper.h>
 #include <prefixtrie.h>
 #include <primitives/transaction.h>
 #include <serialize.h>
@@ -18,11 +18,13 @@
 #include <unordered_set>
 
 // leveldb keys
-#define TRIE_NODE 'n'
+#define TRIE_NODE 'n' // deprecated
+#define TRIE_NODE_BY_HASH 'h'
+#define TRIE_NODE_BY_NAME 'g'
 #define CLAIM_BY_ID 'i'
 #define CLAIM_QUEUE_ROW 'r'
 #define CLAIM_QUEUE_NAME_ROW 'm'
-#define EXP_QUEUE_ROW 'e'
+#define CLAIM_EXP_QUEUE_ROW 'e'
 #define SUPPORT 's'
 #define SUPPORT_QUEUE_ROW 'u'
 #define SUPPORT_QUEUE_NAME_ROW 'p'
@@ -61,6 +63,7 @@ struct CClaimValue
         READWRITE(nAmount);
         READWRITE(nHeight);
         READWRITE(nValidAtHeight);
+        READWRITE(nEffectiveAmount);
     }
 
     bool operator<(const CClaimValue& other) const
@@ -157,17 +160,6 @@ struct CClaimTrieData
     inline void SerializationOp(Stream& s, Operation ser_action)
     {
         READWRITE(hash);
-
-        if (ser_action.ForRead()) {
-            if (s.eof()) {
-                claims.clear();
-                nHeightOfLastTakeover = 0;
-                return;
-            }
-        }
-        else if (claims.empty())
-            return;
-
         READWRITE(claims);
         READWRITE(nHeightOfLastTakeover);
     }
@@ -188,6 +180,30 @@ struct CClaimTrieData
     }
 };
 
+struct CClaimTrieDataNode {
+    CClaimTrieData data;
+    // we're using a vector to avoid RAM thrashing and for faster serialization ops.
+    // We're assuming its data is inserted in order and never modified.
+    std::vector<std::pair<std::string, uint256>> children;
+    bool childrenSerialization = true;
+
+    CClaimTrieDataNode() = default;
+    CClaimTrieDataNode(CClaimTrieDataNode&&) = default;
+    CClaimTrieDataNode(const CClaimTrieDataNode&) = default;
+    CClaimTrieDataNode& operator=(CClaimTrieDataNode&&) = default;
+    CClaimTrieDataNode& operator=(const CClaimTrieDataNode& d) = default;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action)
+    {
+        READWRITE(data);
+        if (childrenSerialization) // wanting constexpr but hoping the compiler is smart enough anyway
+            READWRITE(children);
+    }
+};
+
 struct COutPointHeightType
 {
     COutPoint outPoint;
@@ -301,11 +317,11 @@ struct CClaimsForNameType
     CClaimsForNameType& operator=(const CClaimsForNameType&) = default;
 };
 
-class CClaimTrie : public CPrefixTrie<std::string, CClaimTrieData>
+class CClaimTrie
 {
     int nNextHeight = 0;
     int nProportionalDelayFactor = 0;
-    std::unique_ptr<CDBWrapper> db;
+    std::unique_ptr<CSqlWrapper> db;
 
 public:
     CClaimTrie() = default;
@@ -322,6 +338,19 @@ public:
     friend struct ClaimTrieChainFixture;
     friend class CClaimTrieCacheExpirationFork;
     friend class CClaimTrieCacheNormalizationFork;
+
+    std::size_t getTotalNamesInTrie() const;
+    std::size_t getTotalClaimsInTrie() const;
+    CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const;
+    bool checkConsistency(const uint256& rootHash) const;
+
+    bool contains(const std::string& key) const;
+    bool empty() const;
+    bool find(const uint256& key, CClaimTrieDataNode& node) const;
+    bool find(const std::string& key, CClaimTrieDataNode& node) const;
+
+    std::vector<std::pair<std::string, CClaimTrieDataNode>> nodes(const std::string& key) const;
+    void recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<void(const std::string&, const CClaimTrieDataNode&)> function) const;
 };
 
 struct CClaimTrieProofNode
@@ -381,6 +410,8 @@ typedef std::map<int, expirationQueueRowType> expirationQueueType;
 typedef std::set<CClaimValue> claimIndexClaimListType;
 typedef std::vector<CClaimIndexElement> claimIndexElementListType;
 
+typedef CPrefixTrie<std::string, CClaimTrieData> CClaimPrefixTrie;
+
 class CClaimTrieCacheBase
 {
 public:
@@ -388,7 +419,6 @@ public:
     virtual ~CClaimTrieCacheBase() = default;
 
     uint256 getMerkleHash();
-    bool checkConsistency() const;
 
     bool getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const;
 
@@ -402,10 +432,6 @@ public:
     bool haveSupport(const std::string& name, const COutPoint& outPoint) const;
     bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight);
 
-    std::size_t getTotalNamesInTrie() const;
-    std::size_t getTotalClaimsInTrie() const;
-    CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const;
-
     bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
     bool undoAddClaim(const std::string& name, const COutPoint& outPoint, int nHeight);
 
@@ -441,18 +467,18 @@ public:
     CAmount getEffectiveAmountForClaim(const std::string& name, const uint160& claimId, std::vector<CSupportValue>* supports = nullptr) const;
     CAmount getEffectiveAmountForClaim(const CClaimsForNameType& claims, const uint160& claimId, std::vector<CSupportValue>* supports = nullptr) const;
 
-    CClaimTrie::const_iterator begin() const;
+    CClaimPrefixTrie::const_iterator begin() const;
-    CClaimTrie::const_iterator end() const;
+    CClaimPrefixTrie::const_iterator end() const;
-    CClaimTrie::const_iterator find(const std::string& name) const;
 
-    void dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase = true) const;
+    void dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase = true) const;
+    virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
 
 protected:
     CClaimTrie* base;
-    CClaimTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
+    CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
     std::unordered_set<std::string> namesToCheckForTakeover; // takeover numbers are updated on increment
 
-    uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it);
+    uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it);
 
     virtual bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover);
     virtual bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover);
@@ -460,14 +486,12 @@ protected:
     virtual bool insertSupportIntoMap(const std::string& name, const CSupportValue& support, bool fCheckTakeover);
     virtual bool removeSupportFromMap(const std::string& name, const COutPoint& outPoint, CSupportValue& support, bool fCheckTakeover);
 
-    virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
-
     supportEntryType getSupportsForName(const std::string& name) const;
 
     int getDelayForName(const std::string& name) const;
     virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
 
-    CClaimTrie::iterator cacheData(const std::string& name, bool create = true);
+    CClaimPrefixTrie::iterator cacheData(const std::string& name, bool create = true);
 
     bool getLastTakeoverForName(const std::string& name, uint160& claimId, int& takeoverHeight) const;
 
@@ -499,6 +523,7 @@ private:
     std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
     std::unordered_map<std::string, bool> takeoverWorkaround;
     std::unordered_set<std::string> removalWorkaround;
+    std::unordered_set<std::string> dirtyNodes;
 
     bool shouldUseTakeoverWorkaround(const std::string& key) const;
     void addTakeoverWorkaroundPotential(const std::string& key);
@@ -510,6 +535,8 @@ private:
     bool removeSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
     bool removeClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
 
+    bool validateTrieConsistency(const CBlockIndex* tip);
+
     template <typename T>
     std::pair<const int, std::vector<queueEntryType<T>>>* getQueueCacheRow(int nHeight, bool createIfNotExists = false);
 
@@ -614,6 +641,7 @@ public:
     bool getProofForName(const std::string& name, CClaimTrieProof& proof) override;
     bool getInfoForName(const std::string& name, CClaimValue& claim) const override;
     CClaimsForNameType getClaimsForName(const std::string& name) const override;
+    std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override;
 
 protected:
     bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover) override;
@@ -624,8 +652,6 @@ protected:
 
     int getDelayForName(const std::string& name, const uint160& claimId) const override;
 
-    std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override;
-
 private:
     bool overrideInsertNormalization;
     bool overrideRemoveNormalization;

src/claimtrieforks.cpp

@@ -8,6 +8,7 @@
 #include <boost/locale/conversion.hpp>
 #include <boost/locale/localization_backend.hpp>
 #include <boost/scope_exit.hpp>
+#include <boost/scoped_ptr.hpp>
 
 CClaimTrieCacheExpirationFork::CClaimTrieCacheExpirationFork(CClaimTrie* base)
     : CClaimTrieCacheBase(base)
@@ -60,22 +61,22 @@ bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
     */
 
     //look through db for expiration queues, if we haven't already found it in dirty expiration queue
-    boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
+    CSqlIterator pcursor(*(base->db));
-    for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
+    for (pcursor.SeekToFirst(); pcursor.Valid(); pcursor.Next()) {
         std::pair<uint8_t, int> key;
-        if (!pcursor->GetKey(key))
+        if (!pcursor.GetKey(key))
             continue;
         int height = key.second;
-        if (key.first == EXP_QUEUE_ROW) {
+        if (key.first == CLAIM_EXP_QUEUE_ROW) {
             expirationQueueRowType row;
-            if (pcursor->GetValue(row)) {
+            if (pcursor.GetValue(row)) {
                 reactivateClaim(row, height, increment);
             } else {
                 return error("%s(): error reading expiration queue rows from disk", __func__);
             }
         } else if (key.first == SUPPORT_EXP_QUEUE_ROW) {
             expirationQueueRowType row;
-            if (pcursor->GetValue(row)) {
+            if (pcursor.GetValue(row)) {
                 reactivateSupport(row, height, increment);
             } else {
                 return error("%s(): error reading support expiration queue rows from disk", __func__);
@@ -160,40 +161,48 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insert
     // run the one-time upgrade of all names that need to change
     // it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
 
-    for (auto it = base->begin(); it != base->end(); ++it) {
+    CSqlIterator pcursor(*(base->db));
-        const std::string normalized = normalizeClaimName(it.key(), true);
+    for (pcursor.SeekToFirst(); pcursor.Valid(); pcursor.Next()) {
-        if (normalized == it.key())
+        std::pair<uint8_t, std::string> key;
+        if (!pcursor.GetKey(key) || key.first != TRIE_NODE_BY_NAME)
             continue;
 
-        auto supports = getSupportsForName(it.key());
+        const auto& name = key.second;
+        const std::string normalized = normalizeClaimName(name, true);
+        if (normalized == key.second)
+            continue;
+
+        auto supports = getSupportsForName(name);
         for (auto support : supports) {
             // if it's already going to expire just skip it
             if (support.nHeight + expirationTime() <= nNextHeight)
                 continue;
 
-            assert(removeSupportFromMap(it.key(), support.outPoint, support, false));
+            assert(removeSupportFromMap(name, support.outPoint, support, false));
-            expireSupportUndo.emplace_back(it.key(), support);
+            expireSupportUndo.emplace_back(name, support);
             assert(insertSupportIntoMap(normalized, support, false));
-            insertSupportUndo.emplace_back(it.key(), support.outPoint, -1);
+            insertSupportUndo.emplace_back(name, support.outPoint, -1);
         }
 
         namesToCheckForTakeover.insert(normalized);
 
-        auto cached = cacheData(it.key(), false);
+        auto cached = cacheData(name, false);
         if (!cached || cached->empty())
             continue;
 
-        for (auto claim : it->claims) {
+        auto claimsCopy = cached->claims;
+        auto takeoverHeightCopy = cached->nHeightOfLastTakeover;
+        for (auto claim : claimsCopy) {
             if (claim.nHeight + expirationTime() <= nNextHeight)
                 continue;
 
-            assert(removeClaimFromTrie(it.key(), claim.outPoint, claim, false));
+            assert(removeClaimFromTrie(name, claim.outPoint, claim, false));
-            removeUndo.emplace_back(it.key(), claim);
+            removeUndo.emplace_back(name, claim);
             assert(insertClaimIntoTrie(normalized, claim, true));
-            insertUndo.emplace_back(it.key(), claim.outPoint, -1);
+            insertUndo.emplace_back(name, claim.outPoint, -1);
         }
 
-        takeoverHeightUndo.emplace_back(it.key(), it->nHeightOfLastTakeover);
+        takeoverHeightUndo.emplace_back(name, takeoverHeightCopy);
     }
     return true;
 }

src/init.cpp

@@ -1461,7 +1461,7 @@ bool AppInitMain()
                 pblocktree.reset();
                 pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset));
                 delete pclaimTrie;
-                pclaimTrie = new CClaimTrie(false, fReindex);
+                pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState);
 
                 if (fReset) {
                     pblocktree->WriteReindexing(true);

src/miner.cpp

@@ -207,7 +207,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc
 
     CValidationState state;
     if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) {
-        if (!trieCache.empty() && trieCache.checkConsistency())
+        if (!trieCache.empty())
             trieCache.dumpToLog(trieCache.begin());
         throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state)));
     }

src/prefixtrie.cpp

@@ -174,21 +174,6 @@ std::vector<typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>> CPref
     return ret;
 }
 
-template <typename TKey>
-static std::size_t match(const TKey& a, const TKey& b)
-{
-    std::size_t count = 0;
-    auto ait = a.cbegin(), aend = a.cend();
-    auto bit = b.cbegin(), bend = b.cend();
-    while (ait != aend && bit != bend) {
-        if (*ait != *bit) break;
-        ++count;
-        ++ait;
-        ++bit;
-    }
-    return count;
-}
-
 template <typename TKey, typename TData>
 template <typename TIterator, typename TNode>
 TIterator CPrefixTrie<TKey, TData>::find(const TKey& key, TNode node, TIterator end)

src/prefixtrie.h

@@ -193,4 +193,19 @@ inline bool operator!=(const std::reference_wrapper<std::unique_ptr<T>>& ref, co
     return !(ref == obj);
 }
 
+template <typename TKey>
+static std::size_t match(const TKey& a, const TKey& b)
+{
+    std::size_t count = 0;
+    auto ait = a.cbegin(), aend = a.cend();
+    auto bit = b.cbegin(), bend = b.cend();
+    while (ait != aend && bit != bend) {
+        if (*ait != *bit) break;
+        ++count;
+        ++ait;
+        ++bit;
+    }
+    return count;
+}
+
 #endif // BITCOIN_PREFIXTRIE_H

src/rpc/claimtrie.cpp

@@ -174,48 +174,52 @@ static UniValue getclaimsintrie(const JSONRPCRequest& request)
         throw JSONRPCError(RPC_METHOD_DEPRECATED, msg);
     }
 
+    UniValue ret(UniValue::VARR);
+    uint256 rootHash;
     LOCK(cs_main);
 
     CCoinsViewCache coinsCache(pcoinsTip.get());
     CClaimTrieCache trieCache(pclaimTrie);
 
     if (!request.params.empty()) {
-        CBlockIndex* blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)"));
+        CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)"));
         RollBackTo(blockIndex, coinsCache, trieCache);
     }
+    rootHash = trieCache.getMerkleHash();
 
-    UniValue ret(UniValue::VARR);
+    CClaimTrieDataNode rootNode;
-    for (auto it = trieCache.begin(); it != trieCache.end(); ++it)
+    if (!pclaimTrie->find(rootHash, rootNode))
-    {
+        return ret;
+
+    pclaimTrie->recurseAllHashedNodes("", rootNode, [&ret, &trieCache, &coinsCache](const std::string &name,
+                                                                                    const CClaimTrieDataNode &node) {
         if (ShutdownRequested())
             throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested");
 
         boost::this_thread::interruption_point();
 
-        if (it->empty())
+        if (node.data.empty())
-            continue;
+            return;
 
         UniValue claims(UniValue::VARR);
-        for (auto itClaims = it->claims.cbegin(); itClaims != it->claims.cend(); ++itClaims) {
+        for (auto itClaims = node.data.claims.cbegin(); itClaims != node.data.claims.cend(); ++itClaims) {
             UniValue claim(UniValue::VOBJ);
             claim.pushKV("claimId", itClaims->claimId.GetHex());
             claim.pushKV("txid", itClaims->outPoint.hash.GetHex());
-            claim.pushKV("n", (int)itClaims->outPoint.n);
+            claim.pushKV("n", (int) itClaims->outPoint.n);
             claim.pushKV("amount", ValueFromAmount(itClaims->nAmount));
             claim.pushKV("height", itClaims->nHeight);
-            const Coin& coin = coinsCache.AccessCoin(itClaims->outPoint);
+            const Coin &coin = coinsCache.AccessCoin(itClaims->outPoint);
-            if (coin.IsSpent())
+            if (coin.IsSpent()) {
-            {
+                LogPrintf("%s: the specified txout of %s appears to have been spent\n", __func__,
-                LogPrintf("%s: the specified txout of %s appears to have been spent\n", __func__, itClaims->outPoint.hash.GetHex());
+                          itClaims->outPoint.hash.GetHex());
                 claim.pushKV("error", "Txout spent");
-            }
+            } else {
-            else
-            {
                 int op;
                 std::vector<std::vector<unsigned char> > vvchParams;
-                if (!DecodeClaimScript(coin.out.scriptPubKey, op, vvchParams))
+                if (!DecodeClaimScript(coin.out.scriptPubKey, op, vvchParams)) {
-                {
+                    LogPrintf("%s: the specified txout of %s does not have an claim command\n", __func__,
-                    LogPrintf("%s: the specified txout of %s does not have an claim command\n", __func__, itClaims->outPoint.hash.GetHex());
+                              itClaims->outPoint.hash.GetHex());
                 }
                 claim.pushKV("value", HexStr(vvchParams[1].begin(), vvchParams[1].end()));
             }
@@ -228,11 +232,10 @@ static UniValue getclaimsintrie(const JSONRPCRequest& request)
         }
 
         UniValue nodeObj(UniValue::VOBJ);
-        nodeObj.pushKV("normalized_name", escapeNonUtf8(it.key()));
+        nodeObj.pushKV("normalized_name", escapeNonUtf8(name));
         nodeObj.pushKV("claims", claims);
         ret.push_back(nodeObj);
-    }
+    });
-
     return ret;
 }
 
@@ -258,27 +261,33 @@ static UniValue getnamesintrie(const JSONRPCRequest& request)
             "Result: \n"
             "\"names\"            (array) all names in the trie that have claims\n");
 
-    LOCK(cs_main);
+    uint256 rootHash;
+    {
+        LOCK(cs_main);
 
-    CCoinsViewCache coinsCache(pcoinsTip.get());
+        CCoinsViewCache coinsCache(pcoinsTip.get());
-    CClaimTrieCache trieCache(pclaimTrie);
+        CClaimTrieCache trieCache(pclaimTrie);
 
-    if (!request.params.empty()) {
+        if (!request.params.empty()) {
-        CBlockIndex* blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)"));
+            CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)"));
-        RollBackTo(blockIndex, coinsCache, trieCache);
+            RollBackTo(blockIndex, coinsCache, trieCache);
+        }
+        rootHash = trieCache.getMerkleHash();
     }
-
     UniValue ret(UniValue::VARR);
 
-    for (auto it = trieCache.begin(); it != trieCache.end(); ++it) {
+    CClaimTrieDataNode rootNode;
+    if (!pclaimTrie->find(rootHash, rootNode))
+        return ret;
+
+    pclaimTrie->recurseAllHashedNodes("", rootNode, [&ret](const std::string& name, const CClaimTrieDataNode& node) {
+        if (!node.data.empty())
+            ret.push_back(escapeNonUtf8(name));
         if (ShutdownRequested())
             throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested");
 
         boost::this_thread::interruption_point();
-
+    });
-        if (!it->empty())
-            ret.push_back(escapeNonUtf8(it.key()));
-    }
 
     return ret;
 }
@@ -580,8 +589,7 @@ UniValue gettotalclaimednames(const JSONRPCRequest& request)
             "                                         names in the trie\n"
         );
     LOCK(cs_main);
-    CClaimTrieCache trieCache(pclaimTrie);
+    auto num_names = pclaimTrie->getTotalNamesInTrie();
-    auto num_names = trieCache.getTotalNamesInTrie();
     return int(num_names);
 }
 
@@ -597,8 +605,7 @@ UniValue gettotalclaims(const JSONRPCRequest& request)
             "                                       of active claims\n"
         );
     LOCK(cs_main);
-    CClaimTrieCache trieCache(pclaimTrie);
+    auto num_claims = pclaimTrie->getTotalClaimsInTrie();
-    auto num_claims = trieCache.getTotalClaimsInTrie();
     return int(num_claims);
 }
 
@@ -619,8 +626,7 @@ UniValue gettotalvalueofclaims(const JSONRPCRequest& request)
     bool controlling_only = false;
     if (request.params.size() == 1)
         controlling_only = request.params[0].get_bool();
-    CClaimTrieCache trieCache(pclaimTrie);
+    auto total_amount = pclaimTrie->getTotalValueOfClaimsInTrie(controlling_only);
-    auto total_amount = trieCache.getTotalValueOfClaimsInTrie(controlling_only);
     return ValueFromAmount(total_amount);
 }
 

src/serialize.h

@@ -559,8 +559,8 @@ template<typename Stream, typename K, typename T> void Unserialize(Stream& is, s
 /**
  * map
  */
-template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
+template<typename Stream, typename K, typename T, typename ... Z> void Serialize(Stream& os, const std::map<K, T, Z...>& m);
-template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
+template<typename Stream, typename K, typename T, typename ... Z> void Unserialize(Stream& is, std::map<K, T, Z...>& m);
 
 /**
  * set
@@ -781,20 +781,20 @@ void Unserialize(Stream& is, std::pair<K, T>& item)
 /**
  * map
  */
-template<typename Stream, typename K, typename T, typename Pred, typename A>
+template<typename Stream, typename K, typename T, typename ... Z>
-void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
+void Serialize(Stream& os, const std::map<K, T, Z...>& m)
 {
     WriteCompactSize(os, m.size());
     for (const auto& entry : m)
         Serialize(os, entry);
 }
 
-template<typename Stream, typename K, typename T, typename Pred, typename A>
+template<typename Stream, typename K, typename T, typename ... Z>
-void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
+void Unserialize(Stream& is, std::map<K, T, Z...>& m)
 {
     m.clear();
     unsigned int nSize = ReadCompactSize(is);
-    typename std::map<K, T, Pred, A>::iterator mi = m.begin();
+    typename std::map<K, T, Z...>::iterator mi = m.begin();
     for (unsigned int i = 0; i < nSize; i++)
     {
         std::pair<K, T> item;

src/sqlwrapper.h

@@ -0,0 +1,323 @@
+// Copyright (c) 2012-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#pragma once
+
+#include <clientversion.h>
+#include <fs.h>
+#include <serialize.h>
+#include <streams.h>
+#include <util.h>
+#include <utilstrencodings.h>
+#include <version.h>
+
+#include <sqlite3.h>
+
+class CSqlBatch;
+class CSqlIterator;
+
+class CSqlWrapper
+{
+    friend CSqlBatch;
+    friend CSqlIterator;
+    sqlite3 *pdb;
+    mutable CDataStream ssKey;
+    mutable CDataStream ssValue;
+
+    void VerifyRC(int rc, int target, int line) const {
+        if (rc != target) {
+            auto error = sqlite3_errmsg(pdb);
+            LogPrintf("Query error %d on line %d: %s\n", rc, line, error);
+            assert(rc == target);
+        }
+    }
+
+public:
+    /**
+     * @param[in] path        Location in the filesystem where leveldb data will be stored.
+     * @param[in] nCacheSize  Configures various leveldb cache settings.
+     * @param[in] fMemory     If true, use leveldb's memory environment.
+     * @param[in] fWipe       If true, remove all existing data.
+     * @param[in] obfuscate   If true, store data obfuscated via simple XOR. If false, XOR
+     *                        with a zero'd byte array.
+     */
+    CSqlWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false):
+            ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) {
+        int rc = sqlite3_open_v2(fMemory ? ":memory:" : path.c_str(), &pdb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
+        if (rc != SQLITE_OK)
+        {
+            LogPrintf("ERROR: Unable to open %s. Message: %s\n", path, sqlite3_errmsg(pdb));
+            assert(rc == SQLITE_OK);
+        }
+        char* error = nullptr;
+        rc = sqlite3_exec(pdb, "CREATE TABLE IF NOT EXISTS kv (key BLOB PRIMARY KEY, value BLOB);", nullptr, nullptr, &error);
+        if (rc != SQLITE_OK) {
+            LogPrintf("ERROR: Unable to create kv table. Message: %s\n", error);
+            sqlite3_free(error);
+            sqlite3_close(pdb);
+            assert(rc == SQLITE_OK);
+        }
+
+        if (fWipe) {
+            rc = sqlite3_exec(pdb, "DELETE FROM kv;", nullptr, nullptr, &error);
+            if (rc != SQLITE_OK) {
+                LogPrintf("ERROR: Unable to drop kv table. Message: %s\n", error);
+                sqlite3_free(error);
+                sqlite3_close(pdb);
+                assert(rc == SQLITE_OK);
+            }
+        }
+        std::string pragmas = "PRAGMA cache_size=-" + std::to_string(nCacheSize >> 10)
+                + "; PRAGMA journal_mode=WAL; PRAGMA temp_store=MEMORY; PRAGMA synchronous=NORMAL; PRAGMA wal_autocheckpoint=10000;";
+        rc = sqlite3_exec(pdb, pragmas.c_str(), nullptr, nullptr, &error);
+        if (rc != SQLITE_OK) {
+            LogPrintf("ERROR: Unable to set cache size. Message: %s\n", error);
+            sqlite3_free(error);
+            sqlite3_close(pdb);
+            assert(rc == SQLITE_OK);
+        }
+    }
+    ~CSqlWrapper() {
+        sqlite3_close(pdb);
+    }
+
+    CSqlWrapper(const CSqlWrapper&) = delete;
+
+    template <typename K, typename V>
+    bool Read(const K& key, V& value) const
+    {
+        ssKey << key;
+
+        sqlite3_stmt *stmt = nullptr;
+        auto rc = sqlite3_prepare_v2(pdb, "SELECT value FROM kv WHERE key = ?", -1, &stmt, nullptr);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_bind_blob(stmt, 1, ssKey.data(), ssKey.size(), SQLITE_STATIC);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_step(stmt);
+        bool ret = false;
+        if (rc == SQLITE_ROW) {
+            auto blob_size = sqlite3_column_bytes(stmt, 0);
+            auto blob = reinterpret_cast<const char*>(sqlite3_column_blob(stmt, 0)); // released on step/finalize
+            if (blob_size && blob) {
+                try {
+                    CDataStream ssValue(blob, blob + blob_size, SER_DISK, CLIENT_VERSION);
+                    ssValue >> value;
+                    ret = true;
+                } catch (const std::exception &) {}
+            }
+        }
+        sqlite3_finalize(stmt);
+        ssKey.clear();
+        return ret;
+    }
+
+    template <typename K, typename V>
+    bool Write(const K& key, const V& value, bool fSync = false)
+    {
+        ssKey << key;
+        ssValue << value;
+
+        sqlite3_stmt *stmt = nullptr;
+        auto rc = sqlite3_prepare_v2(pdb, "REPLACE INTO kv VALUES(?, ?)", -1, &stmt, nullptr);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_bind_blob(stmt, 1, ssKey.data(), ssKey.size(), SQLITE_STATIC);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_bind_blob(stmt, 2, ssValue.data(), ssValue.size(), SQLITE_STATIC);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_step(stmt);
+        VerifyRC(rc, SQLITE_DONE, __LINE__);
+        auto ret = sqlite3_changes(pdb) > 0;
+        sqlite3_finalize(stmt);
+
+        ssKey.clear();
+        ssValue.clear();
+
+        return ret;
+    }
+
+    template <typename K>
+    bool Exists(const K& key) const
+    {
+        ssKey << key;
+
+        sqlite3_stmt *stmt = nullptr;
+        auto rc = sqlite3_prepare_v2(pdb, "SELECT 1 FROM kv WHERE key = ?", -1, &stmt, nullptr);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_bind_blob(stmt, 1, ssKey.data(), ssKey.size(), SQLITE_STATIC);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_step(stmt);
+        bool ret = false;
+        if (rc == SQLITE_ROW)
+            ret = 1 == sqlite3_column_int(stmt, 0);
+        sqlite3_finalize(stmt);
+
+        ssKey.clear();
+        return ret;
+    }
+
+    template <typename K>
+    bool Erase(const K& key, bool fSync = false)
+    {
+        ssKey << key;
+
+        sqlite3_stmt *stmt = nullptr;
+        auto rc = sqlite3_prepare_v2(pdb, "DELETE FROM kv WHERE key = ?", -1, &stmt, nullptr);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_bind_blob(stmt, 1, ssKey.data(), ssKey.size(), SQLITE_STATIC);
+        VerifyRC(rc, SQLITE_OK, __LINE__);
+        rc = sqlite3_step(stmt);
+        VerifyRC(rc, SQLITE_DONE, __LINE__);
+        auto ret = sqlite3_changes(pdb) > 0;
+        sqlite3_finalize(stmt);
+        ssKey.clear();
+        if (fSync) Sync();
+        return ret;
+    }
+
+    bool WriteBatch(CSqlBatch& batch, bool fSync = false) {
+        int rc = sqlite3_exec(pdb, "COMMIT TRANSACTION;", nullptr, nullptr, nullptr);
+        if (rc == SQLITE_OK) {
+            if (fSync)
+                Sync();
+            return true;
+        }
+        return false;
+    }
+
+    // not available for LevelDB; provide for compatibility with BDB
+    bool Flush()
+    {
+        return true;
+    }
+
+    bool Sync()
+    {
+        auto rc = sqlite3_wal_checkpoint_v2(pdb, nullptr, SQLITE_CHECKPOINT_FULL, nullptr, nullptr);
+        return rc == SQLITE_OK;
+    }
+
+    /**
+     * Return true if the database managed by this class contains no entries.
+     */
+    bool IsEmpty() {
+        int64_t count = -1;
+        static auto cb = [](void* state, int argc, char** argv, char** cols) {
+            auto data = reinterpret_cast<int64_t*>(state);
+            if (argc)
+                *data = std::atoll(argv[0]);
+            return SQLITE_OK;
+        };
+        int rc = sqlite3_exec(pdb, "SELECT COUNT(*) FROM kv", cb, &count, nullptr);
+        assert(rc == SQLITE_OK);
+        return count == 0;
+    }
+};
+
+/** Batch of changes queued to be written to a CDBWrapper */
+class CSqlBatch
+{
+    friend class CSqlWrapper;
+
+private:
+    CSqlWrapper &parent;
+
+public:
+    /**
+     * @param[in] _parent   CDBWrapper that this batch is to be submitted to
+     */
+    explicit CSqlBatch(CSqlWrapper &_parent) : parent(_parent) {
+        int rc = sqlite3_exec(parent.pdb, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr);
+        parent.VerifyRC(rc, SQLITE_OK, __LINE__);
+    };
+
+    void Clear()
+    {
+        int rc = sqlite3_exec(parent.pdb, "ROLLBACK TRANSACTION;", nullptr, nullptr, nullptr);
+        parent.VerifyRC(rc, SQLITE_OK, __LINE__);
+    }
+
+    template <typename K, typename V>
+    void Write(const K& key, const V& value)
+    {
+        auto ret = parent.Write(key, value, false);
+        assert(ret);
+    }
+
+    template <typename K>
+    void Erase(const K& key)
+    {
+        parent.Erase(key, false);
+    }
+};
+
+class CSqlIterator
+{
+private:
+    const CSqlWrapper &parent;
+    sqlite3_stmt *stmt;
+    int last_step;
+
+public:
+
+    /**
+     * @param[in] _parent          Parent CDBWrapper instance.
+     * @param[in] _piter           The original leveldb iterator.
+     */
+    CSqlIterator(const CSqlWrapper &_parent) :
+            parent(_parent) {
+        stmt = nullptr;
+        auto rc = sqlite3_prepare_v2(parent.pdb, "SELECT * FROM kv;", -1, &stmt, nullptr);
+        parent.VerifyRC(rc, SQLITE_OK, __LINE__);
+    };
+
+    ~CSqlIterator() {
+        last_step = SQLITE_DONE;
+        sqlite3_finalize(stmt);
+    }
+
+    bool Valid() const { return last_step == SQLITE_ROW; }
+
+    void SeekToFirst() { auto rc = sqlite3_reset(stmt); assert(rc == SQLITE_OK); Next(); }
+
+    void Next() { last_step = sqlite3_step(stmt); }
+
+    template<typename K> bool GetKey(K& key) {
+        if (last_step != SQLITE_ROW)
+            return false;
+
+        auto blob_size = sqlite3_column_bytes(stmt, 0);
+        auto blob = reinterpret_cast<const char*>(sqlite3_column_blob(stmt, 0)); // released on step/finalize
+        if (blob_size && blob) {
+            try {
+                CDataStream ssKey(blob, blob + blob_size, SER_DISK, CLIENT_VERSION);
+                ssKey >> key;
+                return true;
+            } catch (const std::exception &) {}
+        }
+        return false;
+    }
+
+    template<typename V> bool GetValue(V& value) {
+        if (last_step != SQLITE_ROW)
+            return false;
+
+        auto blob_size = sqlite3_column_bytes(stmt, 1);
+        auto blob = reinterpret_cast<const char*>(sqlite3_column_blob(stmt, 1)); // released on step/finalize
+        if (blob_size && blob) {
+            try {
+                CDataStream ssValue(blob, blob + blob_size, SER_DISK, CLIENT_VERSION);
+                ssValue >> value;
+                return true;
+            } catch (const std::exception &) {}
+        }
+        return false;
+    }
+
+    int GetValueSize() {
+        if (last_step != SQLITE_ROW)
+            return -1;
+        return sqlite3_column_bytes(stmt, 1);
+    }
+};
+

src/test/claimtriebranching_tests.cpp

@@ -322,16 +322,16 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork
     template <typename K>
     bool keyTypeEmpty(uint8_t keyType)
     {
-        boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
+        CSqlIterator pcursor(*(base->db));
-        pcursor->SeekToFirst();
+        pcursor.SeekToFirst();
 
-        while (pcursor->Valid()) {
+        while (pcursor.Valid()) {
             std::pair<uint8_t, K> key;
-            if (pcursor->GetKey(key)) {
+            if (pcursor.GetKey(key)) {
                 if (key.first == keyType)
                     return false;
             }
-            pcursor->Next();
+            pcursor.Next();
         }
         return true;
     }
@@ -351,7 +351,7 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork
             if (!expirationQueue.second.empty())
                 return false;
         }
-        return keyTypeEmpty<int>(EXP_QUEUE_ROW);
+        return keyTypeEmpty<int>(CLAIM_EXP_QUEUE_ROW);
     }
 
     bool supportEmpty()
@@ -431,6 +431,8 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork
             }
         }
     }
+
+    std::size_t getTotalNamesInTrie() const { return base->getTotalNamesInTrie(); }
 };
 
 BOOST_FIXTURE_TEST_SUITE(claimtriebranching_tests, RegTestingSetup)
@@ -495,6 +497,8 @@ BOOST_AUTO_TEST_CASE(triehash_fuzzer_test)
             }
         }
         fixture.IncrementBlocks(1);
+        if (blocks > 13 && i % 50 == 0) // travisCI needs some periodic output
+            std::cerr << "In triehash_fuzzer_test with " << fixture.getTotalNamesInTrie() << " names at block " << i << std::endl;
     }
 
     if (blocks == 1000 && claimsPerBlock == 100)
@@ -1021,7 +1025,7 @@ BOOST_AUTO_TEST_CASE(claimtrie_expire_test)
     BOOST_CHECK(fixture.is_best_claim("test", tx6));
 }
 /*
- * tests for CClaimTrie::getEffectiveAmountForClaim
+ * tests for CClaimPrefixTrie::getEffectiveAmountForClaim
  */
 BOOST_AUTO_TEST_CASE(claimtriebranching_get_effective_amount_for_claim)
 {
@@ -1066,7 +1070,7 @@ BOOST_AUTO_TEST_CASE(claimtriebranching_get_effective_amount_for_claim)
 }
 
 /*
- * tests for CClaimTrie::getClaimById basic consistency checks
+ * tests for CClaimPrefixTrie::getClaimById basic consistency checks
  */
 BOOST_AUTO_TEST_CASE(get_claim_by_id_test)
 {
@@ -1544,10 +1548,8 @@ BOOST_AUTO_TEST_CASE(claimtriecache_normalization)
     BOOST_CHECK(trieCache.incrementBlock(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo, takeoverHeightUndo));
     BOOST_CHECK(trieCache.shouldNormalize());
 
-    // we cannot use getXXXForName cause they will normalized name
+    CClaimTrieDataNode node;
-    for (auto it = pclaimTrie->begin(); it != pclaimTrie->end(); ++it) {
+    BOOST_CHECK(!pclaimTrie->find(name, node));
-        BOOST_CHECK(it.key() != name);
-    }
 }
 
 BOOST_AUTO_TEST_CASE(undo_normalization_does_not_kill_claim_order)
@@ -2183,7 +2185,7 @@ BOOST_AUTO_TEST_CASE(insert_update_claim_test)
                                          << std::vector<unsigned char>(sName1.begin(), sName1.end())
                                          << std::vector<unsigned char>(tx1ClaimId.begin(), tx1ClaimId.end())
                                          << std::vector<unsigned char>(sValue1.begin(), sValue1.end()) << OP_2DROP << OP_2DROP << OP_TRUE;
-    tx8.vout[0].nValue = tx8.vout[0].nValue - 1;
+    tx8.vout[0].nValue -= 1;
     tx8.vout[1].scriptPubKey = CScript() << OP_CLAIM_NAME
                                          << std::vector<unsigned char>(sName1.begin(), sName1.end())
                                          << std::vector<unsigned char>(sValue2.begin(), sValue2.end()) << OP_2DROP << OP_DROP << OP_TRUE;
@@ -3817,7 +3819,7 @@ BOOST_AUTO_TEST_CASE(bogus_claimtrie_hash_test)
 }
 
 /*
- * tests for CClaimTrie::getClaimById basic consistency checks
+ * tests for CClaimPrefixTrie::getClaimById basic consistency checks
  */
 BOOST_AUTO_TEST_CASE(get_claim_by_id_test_2)
 {
@@ -4099,14 +4101,17 @@ BOOST_AUTO_TEST_CASE(update_on_support2_test)
     CMutableTransaction s2 = fixture.MakeSupport(fixture.GetCoinbase(), tx1, name, 1);
     fixture.IncrementBlocks(1);
 
-    BOOST_CHECK_EQUAL(pclaimTrie->find(name)->nHeightOfLastTakeover, height + 1);
+    CClaimTrieDataNode node;
+    BOOST_CHECK(pclaimTrie->find(name, node));
+    BOOST_CHECK_EQUAL(node.data.nHeightOfLastTakeover, height + 1);
 
     fixture.Spend(s1);
     fixture.Spend(s2);
     CMutableTransaction u1 = fixture.MakeUpdate(tx1, name, value, ClaimIdHash(tx1.GetHash(), 0), 3);
     fixture.IncrementBlocks(1);
 
-    BOOST_CHECK_EQUAL(pclaimTrie->find(name)->nHeightOfLastTakeover, height + 1);
+    BOOST_CHECK(pclaimTrie->find(name, node));
+    BOOST_CHECK_EQUAL(node.data.nHeightOfLastTakeover, height + 1);
 }
 
 BOOST_AUTO_TEST_SUITE_END()

src/test/claimtriecache_tests.cpp

@@ -36,7 +36,7 @@ public:
         return nodesToAddOrUpdate.height();
     }
 
-    CClaimTrie::iterator getCache(const std::string& key)
+    CClaimPrefixTrie::iterator getCache(const std::string& key)
     {
         return nodesToAddOrUpdate.find(key);
     }
@@ -124,7 +124,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
 
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
-    BOOST_CHECK(ntState.checkConsistency());
+    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
 
     CClaimTrieCacheTest ntState1(pclaimTrie);
     ntState1.removeClaimFromTrie(std::string("test"), tx1OutPoint, unused, true);
@@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
 
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3);
-    BOOST_CHECK(ntState2.checkConsistency());
+    BOOST_CHECK(pclaimTrie->checkConsistency(hash3));
 
     CClaimTrieCacheTest ntState3(pclaimTrie);
     ntState3.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, hash160, 50, 100, 200), true);
@@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState3.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4);
-    BOOST_CHECK(ntState3.checkConsistency());
+    BOOST_CHECK(pclaimTrie->checkConsistency(hash4));
 
     CClaimTrieCacheTest ntState4(pclaimTrie);
     ntState4.removeClaimFromTrie(std::string("abab"), tx6OutPoint, unused, true);
@@ -160,7 +160,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState4.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
-    BOOST_CHECK(ntState4.checkConsistency());
+    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
 
     CClaimTrieCacheTest ntState5(pclaimTrie);
     ntState5.removeClaimFromTrie(std::string("test"), tx3OutPoint, unused, true);
@@ -169,7 +169,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState5.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
-    BOOST_CHECK(ntState5.checkConsistency());
+    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
 
     CClaimTrieCacheTest ntState6(pclaimTrie);
     ntState6.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, hash160, 50, 101, 201), true);
@@ -178,7 +178,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState6.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
-    BOOST_CHECK(ntState6.checkConsistency());
+    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
 
     CClaimTrieCacheTest ntState7(pclaimTrie);
     ntState7.removeClaimFromTrie(std::string("test"), tx3OutPoint, unused, true);
@@ -190,7 +190,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState7.flush();
     BOOST_CHECK(pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
-    BOOST_CHECK(ntState7.checkConsistency());
+    BOOST_CHECK(pclaimTrie->checkConsistency(hash0));
 }
 
 BOOST_AUTO_TEST_CASE(basic_insertion_info_test)
@@ -299,28 +299,12 @@ BOOST_AUTO_TEST_CASE(iteratetrie_test)
     BOOST_CHECK(ctc.flush());
 
     std::size_t count = 0;
-    for (auto it = pclaimTrie->begin(); it != pclaimTrie->end(); ++it) {
+    CClaimTrieDataNode node;
-        ++count;
+    BOOST_CHECK(pclaimTrie->find("", node));
-        if (it.key() == "test") {
+    BOOST_CHECK_EQUAL(node.children.size(), 1U);
-            BOOST_CHECK_EQUAL(it->claims.size(), 1);
+    BOOST_CHECK(pclaimTrie->find("test", node));
-        }
+    BOOST_CHECK_EQUAL(node.children.size(), 0U);
-    }
+    BOOST_CHECK_EQUAL(node.data.claims.size(), 1);
-    BOOST_CHECK_EQUAL(count, 2);
-
-    count = 0;
-    for (const auto& it: *pclaimTrie) {
-        ++count;
-        if (it.first == "test") {
-            const CClaimTrieData& data = it.second;
-            BOOST_CHECK_EQUAL(data.claims.size(), 1);
-        }
-    }
-    BOOST_CHECK_EQUAL(count, 2);
-
-    auto it = pclaimTrie->find("test");
-    BOOST_CHECK(it != pclaimTrie->end());
-    BOOST_CHECK_EQUAL(it->claims.size(), 1);
-    BOOST_CHECK_EQUAL(pclaimTrie->height(), 1);
 }
 
 BOOST_AUTO_TEST_CASE(trie_stays_consistent_test)
@@ -337,15 +321,15 @@ BOOST_AUTO_TEST_CASE(trie_stays_consistent_test)
         BOOST_CHECK(cache.insertClaimIntoTrie(name, value, false));
 
     cache.flush();
-    BOOST_CHECK(cache.checkConsistency());
+    BOOST_CHECK(trie.checkConsistency(cache.getMerkleHash()));
 
     for (auto& name: names) {
         CClaimValue temp;
         BOOST_CHECK(cache.removeClaimFromTrie(name, COutPoint(), temp, false));
         cache.flush();
-        BOOST_CHECK(cache.checkConsistency());
+        BOOST_CHECK(trie.checkConsistency(cache.getMerkleHash()));
     }
-    BOOST_CHECK_EQUAL(trie.height(), 0);
+    BOOST_CHECK(trie.empty());
 }
 
 BOOST_AUTO_TEST_CASE(takeover_workaround_triggers)
@@ -394,9 +378,29 @@ BOOST_AUTO_TEST_CASE(takeover_workaround_triggers)
 
     BOOST_CHECK(cache.incrementBlock(icu, ecu, isu, esu, thu));
 
-    BOOST_CHECK_EQUAL(3, cache.find("aa")->nHeightOfLastTakeover);
+    BOOST_CHECK_EQUAL(3, cache.getCache("aa")->nHeightOfLastTakeover);
-    BOOST_CHECK_EQUAL(3, cache.find("bb")->nHeightOfLastTakeover);
+    BOOST_CHECK_EQUAL(3, cache.getCache("bb")->nHeightOfLastTakeover);
-    BOOST_CHECK_EQUAL(1, cache.find("cc")->nHeightOfLastTakeover);
+    BOOST_CHECK_EQUAL(1, cache.getCache("cc")->nHeightOfLastTakeover);
+}
+
+BOOST_AUTO_TEST_CASE(verify_basic_serialization)
+{
+    CClaimValue cv;
+    cv.outPoint = COutPoint(uint256S("123"), 2);
+    cv.nHeight = 3;
+    cv.claimId.SetHex("4567");
+    cv.nEffectiveAmount = 4;
+    cv.nAmount = 5;
+    cv.nValidAtHeight = 6;
+
+    CDataStream ssData(SER_NETWORK, PROTOCOL_VERSION);
+    ssData << cv;
+
+    CClaimValue cv2;
+    ssData >> cv2;
+
+    BOOST_CHECK_EQUAL(cv, cv2);
+    BOOST_CHECK_EQUAL(cv.nEffectiveAmount, cv2.nEffectiveAmount);
 }
 
 BOOST_AUTO_TEST_SUITE_END()

src/test/test_bitcoin.cpp

@@ -20,6 +20,8 @@
 #include <script/sigcache.h>
 
 #include "claimtrie.h"
+#include <boost/test/unit_test.hpp>
+#include <boost/test/unit_test_parameters.hpp>
 
 void CConnmanTest::AddNode(CNode& node)
 {
@@ -39,7 +41,6 @@ void CConnmanTest::ClearNodes()
 uint256 insecure_rand_seed = GetRandHash();
 FastRandomContext insecure_rand_ctx(insecure_rand_seed);
 
-extern bool fPrintToConsole;
 extern void noui_connect();
 
 std::ostream& operator<<(std::ostream& os, const uint256& num)
@@ -90,6 +91,14 @@ std::ostream& operator<<(std::ostream& os, const CSupportValue& support)
 BasicTestingSetup::BasicTestingSetup(const std::string& chainName)
     : m_path_root(fs::temp_directory_path() / "test_bitcoin" / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30))))
 {
+    // for debugging:
+    if (boost::unit_test::runtime_config::get<boost::unit_test::log_level>(boost::unit_test::runtime_config::btrt_log_level)
+            <= boost::unit_test::log_level::log_messages) {
+        g_logger->m_print_to_console = true;
+        g_logger->m_log_time_micros = true;
+        g_logger->EnableCategory(BCLog::ALL);
+    }
+
     SHA256AutoDetect();
     RandomInit();
     ECC_Start();

src/test/test_bitcoin.h

@@ -44,7 +44,7 @@ struct BasicTestingSetup {
     ECCVerifyHandle globalVerifyHandle;
 
     explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN);
-    ~BasicTestingSetup();
+    virtual ~BasicTestingSetup();
 
     fs::path SetDataDir(const std::string& name);
 
@@ -70,7 +70,7 @@ struct TestingSetup: public BasicTestingSetup {
     std::unique_ptr<PeerLogicValidation> peerLogic;
 
     explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN);
-    ~TestingSetup();
+    ~TestingSetup() override;
 };
 
 class CBlock;
@@ -89,7 +89,7 @@ struct TestChain100Setup : public TestingSetup {
     CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns,
                                  const CScript& scriptPubKey);
 
-    ~TestChain100Setup();
+    ~TestChain100Setup() override;
 
     std::vector<CTransactionRef> m_coinbase_txns; // For convenience, coinbase transactions
     CKey coinbaseKey; // private/public key needed to spend coinbase transactions
@@ -97,7 +97,7 @@ struct TestChain100Setup : public TestingSetup {
 
 struct RegTestingSetup: public TestingSetup {
     RegTestingSetup();
-    ~RegTestingSetup();
+    ~RegTestingSetup() override;
 };
 
 class CTxMemPoolEntry;

src/validation.cpp

@@ -1562,13 +1562,11 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI
     assert(trieCache.finalizeDecrement(blockUndo.takeoverHeightUndo));
     auto merkleHash = trieCache.getMerkleHash();
     if (merkleHash != pindex->pprev->hashClaimTrie) {
-        if (trieCache.checkConsistency()) {
+        for (auto cit = trieCache.begin(); cit != trieCache.end(); ++cit) {
-            for (auto cit = trieCache.begin(); cit != trieCache.end(); ++cit) {
+            if (cit->claims.size() && cit->nHeightOfLastTakeover <= 0)
-                if (cit->claims.size() && cit->nHeightOfLastTakeover <= 0)
+                LogPrintf("Invalid takeover height discovered in cache for %s\n", cit.key());
-                    LogPrintf("Invalid takeover height discovered in cache for %s\n", cit.key());
+            if (cit->hash.IsNull())
-                if (cit->hash.IsNull())
+                LogPrintf("Invalid hash discovered in cache for %s\n", cit.key());
-                    LogPrintf("Invalid hash discovered in cache for %s\n", cit.key());
-            }
         }
         LogPrintf("Hash comparison failure at block %d\n", pindex->nHeight);
         assert(merkleHash == pindex->pprev->hashClaimTrie);
@@ -2048,7 +2046,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
 
     if (trieCache.getMerkleHash() != block.hashClaimTrie)
     {
-        if (!trieCache.empty() && trieCache.checkConsistency())
+        if (!trieCache.empty()) // we could run checkConsistency here, but it would take a while
             trieCache.dumpToLog(trieCache.begin());
         return state.DoS(100, error("ConnectBlock() : the merkle root of the claim trie does not match "
                                "(actual=%s vs block=%s on height=%d)", trieCache.getMerkleHash().GetHex(),
@@ -2296,13 +2294,13 @@ void static UpdateTip(const CBlockIndex *pindexNew, const CChainParams& chainPar
     auto oldBlock = pindexNew->nTime + MAX_FUTURE_BLOCK_TIME < currentTime;
     if (!warningMessages.empty() || !oldBlock || lastBlockPrintTime < currentTime - 15 || LogAcceptCategory(BCLog::CLAIMS)) {
         lastBlockPrintTime = currentTime;
-        LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g txb=%lu tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo) trie nodes=%zu%s",
+        LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g txb=%lu tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s",
                 __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion,
                 log(pindexNew->nChainWork.getdouble()) / log(2.0), (unsigned long) pindexNew->nTx,
                 (unsigned long) pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()),
                 GuessVerificationProgress(chainParams.TxData(), pindexNew),
                 pcoinsTip->DynamicMemoryUsage() * (1.0 / (1U << 20U)), pcoinsTip->GetCacheSize(),
-                pclaimTrie->height(), oldBlock ? " (synchronizing)" : "");
+                oldBlock ? " (synchronizing)" : "");
         if (!warningMessages.empty())
             LogPrintf(" warning='%s'", warningMessages); /* Continued */
         LogPrintf("\n");

src/wallet/db.cpp

@@ -154,6 +154,7 @@ BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir
 
 BerkeleyEnvironment::~BerkeleyEnvironment()
 {
+    LOCK(cs_db);
     g_dbenvs.erase(strPath);
     Close();
 }
@@ -558,6 +559,7 @@ BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bo
 
 void BerkeleyBatch::Flush()
 {
+    LOCK(cs_db);
     if (activeTxn)
         return;
 
@@ -566,7 +568,9 @@ void BerkeleyBatch::Flush()
     if (fReadOnly)
         nMinutes = 1;
 
-    env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
+    if (env && env->dbenv) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault
+        env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
+    }
 }
 
 void BerkeleyDatabase::IncrementUpdateCounter()

src/wallet/test/claim_rpc_tests.cpp

@@ -28,11 +28,11 @@ struct CatchWalletTestSetup: public TestingSetup {
         UniValue results = rpc_method(req);
         BOOST_CHECK_EQUAL(results["name"].get_str(), "tester_wallet");
     }
-    ~CatchWalletTestSetup() {
+    ~CatchWalletTestSetup() override {
         rpcfn_type rpc_method = tableRPC["unloadwallet"]->actor;
         JSONRPCRequest req;
+        req.URI = "/wallet/tester_wallet";
         req.params = UniValue(UniValue::VARR);
-        req.params.push_back("tester_wallet");
         rpc_method(req);
     }
 };
@@ -42,6 +42,7 @@ BOOST_FIXTURE_TEST_SUITE(claim_rpc_tests, CatchWalletTestSetup)
 double AvailableBalance() {
     rpcfn_type rpc_method = tableRPC["getbalance"]->actor;
     JSONRPCRequest req;
+    req.URI = "/wallet/tester_wallet";
     req.params = UniValue(UniValue::VARR);
     UniValue results = rpc_method(req);
     return results.get_real();
@@ -51,6 +52,7 @@ uint256 ClaimAName(const std::string& name, const std::string& data, const std::
     // pass a txid as name for update
     rpcfn_type rpc_method = tableRPC[isUpdate ? "updateclaim" : "claimname"]->actor;
     JSONRPCRequest req;
+    req.URI = "/wallet/tester_wallet";
     req.params = UniValue(UniValue::VARR);
     req.params.push_back(name);
     req.params.push_back(data);
@@ -67,6 +69,7 @@ uint256 SupportAName(const std::string& name, const std::string& claimId, const
     // pass a txid as name for update
     rpcfn_type rpc_method = tableRPC["supportclaim"]->actor;
     JSONRPCRequest req;
+    req.URI = "/wallet/tester_wallet";
     req.params = UniValue(UniValue::VARR);
     req.params.push_back(name);
     req.params.push_back(claimId);
@@ -82,6 +85,7 @@ uint256 SupportAName(const std::string& name, const std::string& claimId, const
 UniValue LookupAllNames() {
     rpcfn_type rpc_method = tableRPC["listnameclaims"]->actor;
     JSONRPCRequest req;
+    req.URI = "/wallet/tester_wallet";
     req.params = UniValue(UniValue::VARR);
     return rpc_method(req);
 }
@@ -89,6 +93,7 @@ UniValue LookupAllNames() {
 std::vector<uint256> generateBlock(int blocks = 1) {
     rpcfn_type rpc_method = tableRPC["generate"]->actor;
     JSONRPCRequest req;
+    req.URI = "/wallet/tester_wallet";
     req.params = UniValue(UniValue::VARR);
     req.params.push_back(blocks);
     UniValue results = rpc_method(req);
@@ -107,6 +112,7 @@ void rollbackBlock(const std::vector<uint256>& ids) {
     rpcfn_type rpc_method = tableRPC["invalidateblock"]->actor;
     for (auto it = ids.rbegin(); it != ids.rend(); ++it) {
         JSONRPCRequest req;
+        req.URI = "/wallet/tester_wallet";
         req.params = UniValue(UniValue::VARR);
         req.params.push_back(it->GetHex());
         rpc_method(req);
@@ -119,6 +125,7 @@ void rollbackBlock(const std::vector<uint256>& ids) {
 uint256 AbandonAClaim(const uint256& txid, bool isSupport = false) {
     rpcfn_type pre_rpc_method = tableRPC["getrawchangeaddress"]->actor;
     JSONRPCRequest pre_req;
+    pre_req.URI = "/wallet/tester_wallet";
     pre_req.params = UniValue(UniValue::VARR);
     pre_req.params.push_back("legacy");
     UniValue adr_hash = pre_rpc_method(pre_req);
@@ -126,6 +133,7 @@ uint256 AbandonAClaim(const uint256& txid, bool isSupport = false) {
     // pass a txid as name for update
     rpcfn_type rpc_method = tableRPC[isSupport ? "abandonsupport" : "abandonclaim"]->actor;
     JSONRPCRequest req;
+    req.URI = "/wallet/tester_wallet";
     req.params = UniValue(UniValue::VARR);
     req.params.push_back(txid.GetHex());
     req.params.push_back(adr_hash.get_str());