Use memory mapped file for claim data allocations

Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>

match previous serialization


tweaks


added check for RC's data

src/claimtrie.cpp

@@ -208,15 +208,15 @@ COptional<const std::vector<queueEntryType<T>>> CClaimTrieCacheBase::getQueueCac
 }
 
 template <>
-queueNameRowType* CClaimTrieCacheBase::getQueueCacheNameRow<CClaimValue>(const std::string& name, bool createIfNotExists)
+queueNameRowType* CClaimTrieCacheBase::getQueueCacheNameRow<CClaimValue>(const std::string& name, bool createIfNoExists)
 {
-    return getQueue(*(base->db), CLAIM_QUEUE_NAME_ROW, name, claimQueueNameCache, createIfNotExists);
+    return getQueue(*(base->db), CLAIM_QUEUE_NAME_ROW, name, claimQueueNameCache, createIfNoExists);
 }
 
 template <>
-queueNameRowType* CClaimTrieCacheBase::getQueueCacheNameRow<CSupportValue>(const std::string& name, bool createIfNotExists)
+queueNameRowType* CClaimTrieCacheBase::getQueueCacheNameRow<CSupportValue>(const std::string& name, bool createIfNoExists)
 {
-    return getQueue(*(base->db), SUPPORT_QUEUE_NAME_ROW, name, supportQueueNameCache, createIfNotExists);
+    return getQueue(*(base->db), SUPPORT_QUEUE_NAME_ROW, name, supportQueueNameCache, createIfNoExists);
 }
 
 template <typename T>
@@ -246,15 +246,15 @@ COptional<const queueNameRowType> CClaimTrieCacheBase::getQueueCacheNameRow(cons
 }
 
 template <>
-expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow<CClaimValue>(int nHeight, bool createIfNotExists)
+expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow<CClaimValue>(int nHeight, bool createIfNoExists)
 {
-    return getQueue(*(base->db), CLAIM_EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists);
+    return getQueue(*(base->db), CLAIM_EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNoExists);
 }
 
 template <>
-expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow<CSupportValue>(int nHeight, bool createIfNotExists)
+expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow<CSupportValue>(int nHeight, bool createIfNoExists)
 {
-    return getQueue(*(base->db), SUPPORT_EXP_QUEUE_ROW, nHeight, supportExpirationQueueCache, createIfNotExists);
+    return getQueue(*(base->db), SUPPORT_EXP_QUEUE_ROW, nHeight, supportExpirationQueueCache, createIfNoExists);
 }
 
 template <typename T>
@@ -266,13 +266,8 @@ expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow(int, boo
 
 bool CClaimTrieCacheBase::haveClaim(const std::string& name, const COutPoint& outPoint) const
 {
-    auto it = nodesToAddOrUpdate.find(name);
+    auto it = find(name);
-    if (it && it->haveClaim(outPoint))
+    return it && it->haveClaim(outPoint);
-        return true;
-    if (it || nodesToDelete.count(name))
-        return false;
-    CClaimTrieData data;
-    return base->find(name, data) && data.haveClaim(outPoint);
 }
 
 bool CClaimTrieCacheBase::haveSupport(const std::string& name, const COutPoint& outPoint) const
@@ -325,70 +320,39 @@ bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COut
     return haveInQueue<CSupportValue>(name, outPoint, nValidAtHeight);
 }
 
-void CClaimTrie::recurseNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<recurseNodesCB> function) const {
-    CClaimTrieData data;
-    find(name, data);
-
-    data.hash = current.hash;
-    data.flags |= current.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
-    function(name, data, current.children);
-
-    for (auto& child: current.children) {
-        CClaimTrieDataNode node;
-        auto childName = name + child;
-        if (find(childName, node))
-            recurseNodes(childName, node, function);
-    }
-}
-
 std::size_t CClaimTrie::getTotalNamesInTrie() const
 {
     std::size_t count = 0;
-    CClaimTrieDataNode node;
+    for (auto it = begin(); it != end(); ++it)
-    if (find({}, node))
+        if (!it->empty()) ++count;
-        recurseNodes({}, node, [&count](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
-            count += !data.empty();
-        });
     return count;
 }
 
 std::size_t CClaimTrie::getTotalClaimsInTrie() const
 {
     std::size_t count = 0;
-    CClaimTrieDataNode node;
+    for (auto it = begin(); it != end(); ++it)
-    if (find({}, node))
+        count += it->claims.size();
-        recurseNodes({}, node, [&count]
-        (const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
-            count += data.claims.size();
-        });
     return count;
 }
 
 CAmount CClaimTrie::getTotalValueOfClaimsInTrie(bool fControllingOnly) const
 {
     CAmount value_in_subtrie = 0;
-    CClaimTrieDataNode node;
+    for (auto it = begin(); it != end(); ++it) {
-    if (find({}, node))
+        for (auto& claim : it->claims) {
-        recurseNodes({}, node, [&value_in_subtrie, fControllingOnly]
-        (const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
-            for (const auto &claim : data.claims) {
             value_in_subtrie += claim.nAmount;
             if (fControllingOnly)
                 break;
         }
-        });
+    }
     return value_in_subtrie;
 }
 
 bool CClaimTrieCacheBase::getInfoForName(const std::string& name, CClaimValue& claim) const
 {
-    auto it = nodesToAddOrUpdate.find(name);
+    auto it = find(name);
-    if (it && it->getBestClaim(claim))
+    return it && it->getBestClaim(claim);
-        return true;
-    if (it || nodesToDelete.count(name))
-        return false;
-    CClaimTrieData claims;
-    return base->find(name, claims) && claims.getBestClaim(claim);
 }
 
 template <typename T>
@@ -410,15 +374,9 @@ CClaimSupportToName CClaimTrieCacheBase::getClaimsForName(const std::string& nam
     auto supports = getSupportsForName(name);
     insertRowsFromQueue(supports, name);
 
-    if (auto it = nodesToAddOrUpdate.find(name)) {
+    if (auto it = find(name)) {
         claims = it->claims;
         nLastTakeoverHeight = it->nHeightOfLastTakeover;
-    } else if (!nodesToDelete.count(name)) {
-        CClaimTrieData data;
-        if (base->find(name, data)) {
-            claims = data.claims;
-            nLastTakeoverHeight = data.nHeightOfLastTakeover;
-        }
     }
     insertRowsFromQueue(claims, name);
 
@@ -453,94 +411,79 @@ void completeHash(uint256& partialHash, const std::string& key, std::size_t to)
             .Finalize(partialHash.begin());
 }
 
-bool CClaimTrie::checkConsistency(const uint256& rootHash) const
+template <typename T>
+using iCbType = std::function<void(T&)>;
+
+template <typename TIterator>
+uint256 recursiveMerkleHash(TIterator& it, const iCbType<TIterator>& process)
 {
-    CClaimTrieDataNode node;
-    if (!find({}, node) || node.hash != rootHash) {
-        if (rootHash == one)
-            return true;
-
-        return error("Mismatched root claim trie hashes. This may happen when there is not a clean process shutdown. Please run with -reindex.");
-    }
-
-    bool success = true;
-    recurseNodes({}, node, [&success, this](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
-        if (!success) return;
-
     std::vector<uint8_t> vchToHash;
-        const auto pos = name.size();
+    const auto pos = it.key().size();
-        for (auto &child : children) {
+    for (auto& child : it.children()) {
-            auto key = name + child;
+        process(child);
-            CClaimTrieDataNode node;
+        auto& key = child.key();
-            success &= find(key, node);
+        auto hash = child->hash;
-            auto hash = node.hash;
         completeHash(hash, key, pos);
         vchToHash.push_back(key[pos]);
         vchToHash.insert(vchToHash.end(), hash.begin(), hash.end());
     }
 
     CClaimValue claim;
-        if (data.getBestClaim(claim)) {
+    if (it->getBestClaim(claim)) {
-            uint256 valueHash = getValueHash(claim.outPoint, data.nHeightOfLastTakeover);
+        uint256 valueHash = getValueHash(claim.outPoint, it->nHeightOfLastTakeover);
         vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
-        } else {
+    } else if (!it.hasChildren()) {
-            success &= !children.empty(); // we disallow leaf nodes without claims
+        return {};
-        }
-        success &= data.hash == Hash(vchToHash.begin(), vchToHash.end());
-    });
-
-    return success;
     }
 
-std::vector<std::pair<std::string, CClaimTrieDataNode>> CClaimTrie::nodes(const std::string &key) const {
+    return Hash(vchToHash.begin(), vchToHash.end());
-    std::vector<std::pair<std::string, CClaimTrieDataNode>> ret;
-    CClaimTrieDataNode node;
-
-    if (!find({}, node))
-        return ret;
-
-    ret.emplace_back(std::string{}, node);
-
-    std::string partialKey = key;
-
-    while (!node.children.empty()) {
-        // auto it = node.children.lower_bound(partialKey); // for using a std::map
-        auto it = std::lower_bound(node.children.begin(), node.children.end(), partialKey);
-        if (it != node.children.end() && *it == partialKey) {
-            // we're completely done
-            if (find(key, node))
-                ret.emplace_back(key, node);
-            break;
-        }
-        if (it != node.children.begin()) --it;
-        const auto count = match(partialKey, *it);
-
-        if (count != it->size()) break;
-        if (count == partialKey.size()) break;
-        partialKey = partialKey.substr(count);
-        auto frontKey = key.substr(0, key.size() - partialKey.size());
-        if (find(frontKey, node))
-            ret.emplace_back(frontKey, node);
-        else break;
 }
 
-    return ret;
+bool CClaimTrieCacheBase::recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const
+{
+    struct CRecursiveBreak {};
+    using iterator = CClaimTrie::const_iterator;
+    iCbType<iterator> process = [&failed, &process](iterator& it) {
+        if (it->hash.IsNull() || it->hash != recursiveMerkleHash(it, process)) {
+            failed = it.key();
+            throw CRecursiveBreak();
+        }
+    };
+
+    try {
+        process(it);
+    } catch (const CRecursiveBreak&) {
+        return false;
+    }
+    return true;
 }
 
-bool CClaimTrie::contains(const std::string &key) const {
+bool CClaimTrieCacheBase::checkConsistency() const
-    return db->Exists(std::make_pair(TRIE_NODE_CHILDREN, key));
+{
-}
+    if (base->empty())
+        return true;
 
-bool CClaimTrie::empty() const {
+    auto it = base->cbegin();
-    return !contains({});
+    std::string failed;
+    auto consistent = recursiveCheckConsistency(it, failed);
+    if (!consistent) {
+        LogPrintf("\nPrinting base tree from its parent:\n");
+        auto basePath = base->nodes(failed);
+        if (basePath.size() > 1) basePath.pop_back();
+        dumpToLog(basePath.back(), false);
+        auto cachePath = nodesToAddOrUpdate.nodes(failed);
+        if (!cachePath.empty()) {
+            LogPrintf("\nPrinting %s's parent from cache:\n", failed);
+            if (cachePath.size() > 1) cachePath.pop_back();
+            dumpToLog(cachePath.back(), false);
         }
-
+        if (!nodesToDelete.empty()) {
-bool CClaimTrie::find(const std::string& key, CClaimTrieDataNode &node) const {
+            std::string joined;
-    return db->Read(std::make_pair(TRIE_NODE_CHILDREN, key), node);
+            for (const auto &piece : nodesToDelete) joined += ", " + piece;
+            LogPrintf("Nodes to be deleted: %s\n", joined.substr(2));
         }
-
+    }
-bool CClaimTrie::find(const std::string& key, CClaimTrieData &data) const {
+    return consistent;
-    return db->Read(std::make_pair(TRIE_NODE_CLAIMS, key), data);
 }
 
 template <typename K, typename T>
@@ -579,26 +522,24 @@ bool CClaimTrieCacheBase::flush()
 
     getMerkleHash();
 
+    for (const auto& nodeName : nodesToDelete) {
+        if (nodesToAddOrUpdate.contains(nodeName))
+            continue;
+        auto nodes = base->nodes(nodeName);
+        base->erase(nodeName);
+        for (auto& node : nodes)
+            if (!node)
+                batch.Erase(std::make_pair(TRIE_NODE, node.key()));
+    }
+
     for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) {
-        bool removed = forDeleteFromBase.erase(it.key());
+        auto old = base->find(it.key());
-        if (it->flags & CClaimTrieDataFlags::HASH_DIRTY) {
+        if (!old || old.data() != it.data()) {
-            CClaimTrieDataNode node;
+            base->copy(it);
-            node.hash = it->hash;
+            batch.Write(std::make_pair(TRIE_NODE, it.key()), it.data());
-            for (auto &child: it.children()) // ordering here is important
-                node.children.push_back(child.key().substr(it.key().size()));
-
-            batch.Write(std::make_pair(TRIE_NODE_CHILDREN, it.key()), node);
-
-            if (removed || (it->flags & CClaimTrieDataFlags::CLAIMS_DIRTY))
-                batch.Write(std::make_pair(TRIE_NODE_CLAIMS, it.key()), it.data());
         }
     }
 
-    for (auto& name: forDeleteFromBase) {
-        batch.Erase(std::make_pair(TRIE_NODE_CHILDREN, name));
-        batch.Erase(std::make_pair(TRIE_NODE_CLAIMS, name));
-    }
-
     BatchWriteQueue(batch, SUPPORT, supportCache);
 
     BatchWriteQueue(batch, CLAIM_QUEUE_ROW, claimQueueCache);
@@ -620,32 +561,70 @@ bool CClaimTrieCacheBase::flush()
     return ret;
 }
 
-bool CClaimTrieCacheBase::validateTrieConsistency(const CBlockIndex* tip)
+bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
 {
-    if (!tip || tip->nHeight < 1)
+    LogPrintf("Loading the claim trie from disk...\n");
-        return true;
+
+    base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0;
+
+    if (tip && base->db->Exists(std::make_pair(TRIE_NODE_CHILDREN, std::string()))) {
+        LogPrintf("The claim trie database contains deprecated data and will need to be rebuilt.\n");
+        return false;
+    }
+
+    clear();
+    base->clear();
+    boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
+
+    for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
+        std::pair<uint8_t, std::string> key;
+        if (!pcursor->GetKey(key) || key.first != TRIE_NODE)
+            continue;
+
+        CClaimTrieData data;
+        if (pcursor->GetValue(data)) {
+            if (data.empty()) {
+                // we have a situation where our old trie had many empty nodes
+                // we don't want to automatically throw those all into our prefix trie
+                // we'll run a second pass to clean them up
+                continue;
+            }
+
+            // nEffectiveAmount isn't serialized but it needs to be initialized (as done in reorderClaims):
+            auto supports = getSupportsForName(key.second);
+            data.reorderClaims(supports);
+            base->insert(key.second, std::move(data));
+        } else {
+            return error("%s(): error reading claim trie from disk", __func__);
+        }
+    }
+
+    for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
+        std::pair<uint8_t, std::string> key;
+        if (!pcursor->GetKey(key) || key.first != TRIE_NODE)
+            continue;
+        auto hit = base->find(key.second);
+        if (hit) {
+            CClaimTrieData data;
+            if (pcursor->GetValue(data))
+                hit->hash = data.hash;
+        }
+        else {
+            base->db->Erase(key); // this uses a lot of memory and it's 1-time upgrade from 12.4 so we aren't going to batch it
+        }
+    }
 
     LogPrintf("Checking claim trie consistency... ");
-    if (base->checkConsistency(tip->hashClaimTrie)) {
+    if (checkConsistency()) {
         LogPrintf("consistent\n");
+        if (tip && tip->hashClaimTrie != getMerkleHash())
+            return error("%s(): hashes don't match when reading claimtrie from disk", __func__);
         return true;
     }
     LogPrintf("inconsistent!\n");
     return false;
 }
 
-bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
-{
-    base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0;
-    clear();
-
-    if (tip && base->db->Exists(std::make_pair(TRIE_NODE, std::string()))) {
-        LogPrintf("The claim trie database contains deprecated data and will need to be rebuilt\n");
-        return false;
-    }
-    return validateTrieConsistency(tip);
-}
-
 CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) : base(base)
 {
     assert(base);
@@ -657,92 +636,65 @@ int CClaimTrieCacheBase::expirationTime() const
     return Params().GetConsensus().nOriginalClaimExpirationTime;
 }
 
-uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it)
+uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimTrie::iterator& it)
 {
-    if (!it->hash.IsNull())
+    using iterator = CClaimTrie::iterator;
+    iCbType<iterator> process = [&process](iterator& it) {
+        if (it->hash.IsNull())
+            it->hash = recursiveMerkleHash(it, process);
+        assert(!it->hash.IsNull());
+    };
+    process(it);
     return it->hash;
-
-    std::vector<uint8_t> vchToHash;
-    const auto pos = it.key().size();
-    for (auto& child : it.children()) {
-        auto hash = recursiveComputeMerkleHash(child);
-        auto& key = child.key();
-        completeHash(hash, key, pos);
-        vchToHash.push_back(key[pos]);
-        vchToHash.insert(vchToHash.end(), hash.begin(), hash.end());
-    }
-
-    CClaimValue claim;
-    if (it->getBestClaim(claim)) {
-        uint256 valueHash = getValueHash(claim.outPoint, it->nHeightOfLastTakeover);
-        vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
-    }
-
-    return it->hash = Hash(vchToHash.begin(), vchToHash.end());
 }
 
 uint256 CClaimTrieCacheBase::getMerkleHash()
 {
-    if (auto it = nodesToAddOrUpdate.begin())
+    auto it = nodesToAddOrUpdate.begin();
-        return recursiveComputeMerkleHash(it);
+    if (!it && nodesToDelete.empty())
-    if (nodesToDelete.empty() && nodesAlreadyCached.empty()) {
+        it = base->begin();
-        CClaimTrieDataNode node;
+    return !it ? one : recursiveComputeMerkleHash(it);
-        if (base->find({}, node))
-            return node.hash; // it may be valuable to have base cache its current root hash
-    }
-    return one; // we have no data or we deleted everything
 }
 
-CClaimPrefixTrie::const_iterator CClaimTrieCacheBase::begin() const
+CClaimTrie::const_iterator CClaimTrieCacheBase::find(const std::string& name) const
 {
-    return nodesToAddOrUpdate.begin();
+    auto it = nodesToAddOrUpdate.find(name);
-}
+    if (it || nodesToDelete.count(name))
-
+        return it;
-CClaimPrefixTrie::const_iterator CClaimTrieCacheBase::end() const
+    return base->find(name);
-{
-    return nodesToAddOrUpdate.end();
 }
 
 bool CClaimTrieCacheBase::empty() const
 {
-    return nodesToAddOrUpdate.empty();
+    return nodesToAddOrUpdate.empty(); // only used with the dump method, and we don't want to dump base
 }
 
-CClaimPrefixTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& name, bool create)
+CClaimTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& name, bool create)
 {
+    // get data from the cache. if no data, create empty one
+    const auto insert = [this](CClaimTrie::iterator& it) {
+        auto& key = it.key();
+        // we only ever cache nodes once per cache instance
+        if (!nodesAlreadyCached.count(key)) {
+            // do not insert nodes that are already present
+            nodesAlreadyCached.insert(key);
+            nodesToAddOrUpdate.insert(key, it.data());
+        }
+    };
+
     // we need all parent nodes and their one level deep children
     // to calculate merkle hash
     auto nodes = base->nodes(name);
     for (auto& node: nodes) {
-        if (nodesAlreadyCached.insert(node.first).second) {
+        for (auto& child : node.children())
-            // do not insert nodes that are already present
+            if (!nodesAlreadyCached.count(child.key()))
-            CClaimTrieData data;
+                nodesToAddOrUpdate.copy(child);
-            base->find(node.first, data);
+        insert(node);
-            data.hash = node.second.hash;
-            data.flags = node.second.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
-            nodesToAddOrUpdate.insert(node.first, data);
-        }
-        for (auto& child : node.second.children) {
-            auto childKey = node.first + child;
-            if (nodesAlreadyCached.insert(childKey).second) {
-                CClaimTrieData childData;
-                if (!base->find(childKey, childData))
-                    childData = {};
-                CClaimTrieDataNode childNode;
-                if (base->find(childKey, childNode)) {
-                    childData.hash = childNode.hash;
-                    childData.flags = childNode.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
-                }
-                nodesToAddOrUpdate.insert(childKey, childData);
-            }
-        }
     }
 
     auto it = nodesToAddOrUpdate.find(name);
     if (!it && create) {
         it = nodesToAddOrUpdate.insert(name, CClaimTrieData{});
-        // if (it.hasChildren()) any children should be in the trie (not base alone)
-        //    it->flags |= CClaimTrieDataFlags::POTENTIAL_CHILDREN;
         confirmTakeoverWorkaroundNeeded(name);
     }
 
@@ -763,11 +715,10 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16
         std::tie(claimId, takeoverHeight) = cit->second;
         return true;
     }
-    CClaimTrieData data;
+    if (auto it = base->find(name)) {
-    if (base->find(name, data)) {
+        takeoverHeight = it->nHeightOfLastTakeover;
-        takeoverHeight = data.nHeightOfLastTakeover;
         CClaimValue claim;
-        if (data.getBestClaim(claim)) {
+        if (it->getBestClaim(claim)) {
             claimId = claim.claimId;
             return true;
         }
@@ -777,12 +728,8 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16
 
 void CClaimTrieCacheBase::markAsDirty(const std::string& name, bool fCheckTakeover)
 {
-    for (auto& node : nodesToAddOrUpdate.nodes(name)) {
+    for (auto& node : nodesToAddOrUpdate.nodes(name))
-        node->flags |= CClaimTrieDataFlags::HASH_DIRTY;
         node->hash.SetNull();
-        if (node.key() == name)
-            node->flags |= CClaimTrieDataFlags::CLAIMS_DIRTY;
-    }
 
     if (fCheckTakeover)
         namesToCheckForTakeover.insert(name);
@@ -816,9 +763,6 @@ bool CClaimTrieCacheBase::removeClaimFromTrie(const std::string& name, const COu
         for (auto& child: it.children())
             cacheData(child.key(), false);
 
-        for (auto& node : nodesToAddOrUpdate.nodes(name))
-            forDeleteFromBase.emplace(node.key());
-
         nodesToAddOrUpdate.erase(name);
         nodesToDelete.insert(name);
 
@@ -1055,13 +999,11 @@ bool CClaimTrieCacheBase::removeSupportFromMap(const std::string& name, const CO
     return false;
 }
 
-void CClaimTrieCacheBase::dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase) const
+void CClaimTrieCacheBase::dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase) const
 {
-    if (!it) return;
-
     if (diffFromBase) {
-        CClaimTrieDataNode node;
+        auto hit = base->find(it.key());
-        if (base->find(it.key(), node) && node.hash == it->hash)
+        if (hit && hit->hash == it->hash)
             return;
     }
 
@@ -1375,12 +1317,8 @@ int CClaimTrieCacheBase::getNumBlocksOfContinuousOwnership(const std::string& na
         that->removalWorkaround.erase(hit);
         return 0;
     }
-    if (auto it = nodesToAddOrUpdate.find(name))
+    auto it = nodesToAddOrUpdate.find(name);
-        return it->empty() ? 0 : nNextHeight - it->nHeightOfLastTakeover;
+    return (it || (it = base->find(name))) && !it->empty() ? nNextHeight - it->nHeightOfLastTakeover : 0;
-    CClaimTrieData data;
-    if (base->find(name, data) && !data.empty())
-        return nNextHeight - data.nHeightOfLastTakeover;
-    return 0;
 }
 
 int CClaimTrieCacheBase::getDelayForName(const std::string& name) const
@@ -1407,23 +1345,22 @@ std::string CClaimTrieCacheBase::adjustNameForValidHeight(const std::string& nam
 
 bool CClaimTrieCacheBase::clear()
 {
-    forDeleteFromBase.clear();
-    nodesToAddOrUpdate.clear();
-    claimsToAddToByIdIndex.clear();
     supportCache.clear();
     nodesToDelete.clear();
-    claimsToDeleteFromByIdIndex.clear();
     takeoverCache.clear();
     claimQueueCache.clear();
     supportQueueCache.clear();
+    removalWorkaround.clear();
+    nodesToAddOrUpdate.clear();
     nodesAlreadyCached.clear();
     takeoverWorkaround.clear();
-    removalWorkaround.clear();
     claimQueueNameCache.clear();
     expirationQueueCache.clear();
     supportQueueNameCache.clear();
+    claimsToAddToByIdIndex.clear();
     namesToCheckForTakeover.clear();
     supportExpirationQueueCache.clear();
+    claimsToDeleteFromByIdIndex.clear();
     return true;
 }
 
@@ -1433,7 +1370,7 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, CClaimTriePro
     cacheData(name, false);
     getMerkleHash();
     proof = CClaimTrieProof();
-    for (auto& it : static_cast<const CClaimPrefixTrie&>(nodesToAddOrUpdate).nodes(name)) {
+    for (auto& it : static_cast<const CClaimTrie&>(nodesToAddOrUpdate).nodes(name)) {
         CClaimValue claim;
         const auto& key = it.key();
         bool fNodeHasValue = it->getBestClaim(claim);
@@ -1449,7 +1386,7 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, CClaimTriePro
                 for (auto i = pos; i + 1 < childKey.size(); ++i) {
                     children.emplace_back(childKey[i], uint256{});
                     proof.nodes.emplace_back(children, fNodeHasValue, valueHash);
-                    children.clear(); // move promises to leave it in a valid state only
+                    children.clear();
                     valueHash.SetNull();
                     fNodeHasValue = false;
                 }
@@ -1473,33 +1410,22 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, CClaimTriePro
     return true;
 }
 
-void CClaimTrieCacheBase::recurseNodes(const std::string &name,
+void CClaimTrieCacheBase::iterate(std::function<void(const std::string&, const CClaimTrieData&)> callback) const
-        std::function<void(const std::string &, const CClaimTrieData &)> function) const {
+{
-
+    if (nodesToAddOrUpdate.empty()) {
-    std::function<CClaimTrie::recurseNodesCB> baseFunction = [this, &function]
+        for (auto it = base->cbegin(); it != base->cend(); ++it)
-            (const std::string& name, const CClaimTrieData& data, const std::vector<std::string>&) {
+            if (!nodesToDelete.count(it.key()))
-        if (nodesToDelete.find(name) == nodesToDelete.end())
+                callback(it.key(), it.data());
-            function(name, data);
+        return;
-    };
-
-    if (empty()) {
-        CClaimTrieDataNode node;
-        if (base->find(name, node))
-            base->recurseNodes(name, node, baseFunction);
-    }
-    else {
-        for (auto it = begin(); it != end(); ++it) {
-            function(it.key(), it.data());
-            if ((it->flags & CClaimTrieDataFlags::POTENTIAL_CHILDREN) && !it.hasChildren()) {
-                CClaimTrieDataNode node;
-                if (base->find(it.key(), node))
-                    for (auto& partialKey: node.children) {
-                        auto childKey = it.key() + partialKey;
-                        CClaimTrieDataNode childNode;
-                        if (base->find(childKey, childNode))
-                            base->recurseNodes(childKey, childNode, baseFunction);
-                    }
-            }
     }
+    for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) {
+        callback(it.key(), it.data());
+        if (it.hasChildren() || nodesToDelete.count(it.key()))
+            continue;
+        auto children = base->find(it.key()).children();
+        for (auto& child : children)
+            for (; child; ++child)
+                if (!nodesToDelete.count(child.key()))
+                    callback(child.key(), child.data());
     }
 }

src/claimtrie.h

@@ -18,9 +18,8 @@
 #include <unordered_set>
 
 // leveldb keys
-#define TRIE_NODE 'n' // deprecated
+#define TRIE_NODE 'n'
 #define TRIE_NODE_CHILDREN 'b'
-#define TRIE_NODE_CLAIMS 'c'
 #define CLAIM_BY_ID 'i'
 #define CLAIM_QUEUE_ROW 'r'
 #define CLAIM_QUEUE_NAME_ROW 'm'
@@ -63,7 +62,6 @@ struct CClaimValue
         READWRITE(nAmount);
         READWRITE(nHeight);
         READWRITE(nValidAtHeight);
-        READWRITE(nEffectiveAmount);
     }
 
     bool operator<(const CClaimValue& other) const
@@ -136,21 +134,12 @@ struct CSupportValue
 typedef std::vector<CClaimValue> claimEntryType;
 typedef std::vector<CSupportValue> supportEntryType;
 
-enum CClaimTrieDataFlags: uint32_t {
-    HASH_DIRTY = 1U,
-    CLAIMS_DIRTY = 2U,
-    POTENTIAL_CHILDREN = 4U, // existing on disk
-};
-
 struct CClaimTrieData
 {
+    uint256 hash;
     claimEntryType claims;
     int nHeightOfLastTakeover = 0;
 
-    // non-serialized data:
-    uint32_t flags = 0;
-    uint256 hash;
-
     CClaimTrieData() = default;
     CClaimTrieData(CClaimTrieData&&) = default;
     CClaimTrieData(const CClaimTrieData&) = default;
@@ -168,13 +157,25 @@ struct CClaimTrieData
     template <typename Stream, typename Operation>
     inline void SerializationOp(Stream& s, Operation ser_action)
     {
+        READWRITE(hash);
+
+        if (ser_action.ForRead()) {
+            if (s.eof()) {
+                claims.clear();
+                nHeightOfLastTakeover = 0;
+                return;
+            }
+        }
+        else if (claims.empty())
+            return;
+
         READWRITE(claims);
         READWRITE(nHeightOfLastTakeover);
     }
 
     bool operator==(const CClaimTrieData& other) const
     {
-        return nHeightOfLastTakeover == other.nHeightOfLastTakeover && claims == other.claims;
+        return hash == other.hash && nHeightOfLastTakeover == other.nHeightOfLastTakeover && claims == other.claims;
     }
 
     bool operator!=(const CClaimTrieData& other) const
@@ -188,28 +189,6 @@ struct CClaimTrieData
     }
 };
 
-struct CClaimTrieDataNode {
-    uint256 hash;
-    // we're using a vector to avoid RAM thrashing and for faster serialization ops.
-    // We're assuming its data is inserted in order and never modified.
-    std::vector<std::string> children;
-
-    CClaimTrieDataNode() = default;
-    CClaimTrieDataNode(CClaimTrieDataNode&&) = default;
-    CClaimTrieDataNode(const CClaimTrieDataNode&) = default;
-    CClaimTrieDataNode& operator=(CClaimTrieDataNode&&) = default;
-    CClaimTrieDataNode& operator=(const CClaimTrieDataNode& d) = default;
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(hash);
-        READWRITE(children);
-    }
-};
-
 struct COutPointHeightType
 {
     COutPoint outPoint;
@@ -364,7 +343,7 @@ struct CClaimSupportToName
     const std::vector<CSupportValue> unmatchedSupports;
 };
 
-class CClaimTrie
+class CClaimTrie : public CPrefixTrie<std::string, CClaimTrieData>
 {
 public:
     CClaimTrie() = default;
@@ -387,23 +366,12 @@ public:
 
     std::size_t getTotalNamesInTrie() const;
     std::size_t getTotalClaimsInTrie() const;
-    virtual bool checkConsistency(const uint256& rootHash) const;
     CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const;
 
-    bool contains(const std::string& key) const;
-    bool empty() const;
-    bool find(const std::string& key, CClaimTrieDataNode& node) const;
-    bool find(const std::string& key, CClaimTrieData& claims) const;
-
-    std::vector<std::pair<std::string, CClaimTrieDataNode>> nodes(const std::string& key) const;
-
 protected:
     int nNextHeight = 0;
     int nProportionalDelayFactor = 0;
     std::unique_ptr<CDBWrapper> db;
-
-    using recurseNodesCB = void(const std::string&, const CClaimTrieData&, const std::vector<std::string>&);
-    void recurseNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<recurseNodesCB> function) const;
 };
 
 struct CClaimTrieProofNode
@@ -508,8 +476,6 @@ typedef std::map<int, expirationQueueRowType> expirationQueueType;
 typedef std::set<CClaimValue> claimIndexClaimListType;
 typedef std::vector<CClaimIndexElement> claimIndexElementListType;
 
-typedef CPrefixTrie<std::string, CClaimTrieData> CClaimPrefixTrie;
-
 class CClaimTrieCacheBase
 {
 public:
@@ -520,6 +486,7 @@ public:
 
     bool flush();
     bool empty() const;
+    bool checkConsistency() const;
     bool ReadFromDisk(const CBlockIndex* tip);
 
     bool haveClaim(const std::string& name, const COutPoint& outPoint) const;
@@ -560,21 +527,20 @@ public:
 
     virtual CClaimSupportToName getClaimsForName(const std::string& name) const;
 
-    CClaimPrefixTrie::const_iterator begin() const;
+    CClaimTrie::const_iterator find(const std::string& name) const;
-    CClaimPrefixTrie::const_iterator end() const;
+    void iterate(std::function<void(const std::string&, const CClaimTrieData&)> callback) const;
 
-    void dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase = true) const;
+    void dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase = true) const;
     virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
 
-    void recurseNodes(const std::string& name, std::function<void(const std::string&, const CClaimTrieData&)> function) const;
-
 protected:
     CClaimTrie* base;
-    CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
+    CClaimTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
     std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
     std::unordered_set<std::string> namesToCheckForTakeover; // takeover numbers are updated on increment
 
-    virtual uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it);
+    virtual uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it);
+    virtual bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const;
 
     virtual bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover);
     virtual bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover);
@@ -587,7 +553,7 @@ public:
     int getDelayForName(const std::string& name) const;
     virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
 
-    CClaimPrefixTrie::iterator cacheData(const std::string& name, bool create = true);
+    CClaimTrie::iterator cacheData(const std::string& name, bool create = true);
 
     bool getLastTakeoverForName(const std::string& name, uint160& claimId, int& takeoverHeight) const;
 
@@ -618,7 +584,6 @@ private:
     std::unordered_set<std::string> nodesToDelete; // to be removed from base (and disk) on flush
     std::unordered_map<std::string, bool> takeoverWorkaround;
     std::unordered_set<std::string> removalWorkaround;
-    std::unordered_set<std::string> forDeleteFromBase;
 
     bool shouldUseTakeoverWorkaround(const std::string& key) const;
     void addTakeoverWorkaroundPotential(const std::string& key);
@@ -630,8 +595,6 @@ private:
     bool removeSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
     bool removeClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
 
-    bool validateTrieConsistency(const CBlockIndex* tip);
-
     template <typename T>
     void insertRowsFromQueue(std::vector<T>& result, const std::string& name) const;
 
@@ -781,20 +744,13 @@ public:
     bool allowSupportMetadata() const;
 
 protected:
-    uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it) override;
+    uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it) override;
+    bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const override;
 
 private:
     void copyAllBaseToCache();
 };
 
-class CClaimTrieHashFork : public CClaimTrie
-{
-public:
-    using CClaimTrie::CClaimTrie;
-protected:
-    bool checkConsistency(const uint256& rootHash) const override;
-};
-
 typedef CClaimTrieCacheHashFork CClaimTrieCache;
 
 #endif // BITCOIN_CLAIMTRIE_H

src/claimtrieforks.cpp

@@ -4,8 +4,6 @@
 #include <claimtrie.h>
 #include <hash.h>
 
-#include <boost/algorithm/string.hpp>
-#include <boost/foreach.hpp>
 #include <boost/locale.hpp>
 #include <boost/locale/conversion.hpp>
 #include <boost/locale/localization_backend.hpp>
@@ -174,17 +172,12 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insert
     // run the one-time upgrade of all names that need to change
     // it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
 
-    boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
+    for (auto it = base->cbegin(); it != base->cend(); ++it) {
-    for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
+        const std::string normalized = normalizeClaimName(it.key(), true);
-        std::pair<uint8_t, std::string> key;
+        if (normalized == it.key())
-        if (!pcursor->GetKey(key) || key.first != TRIE_NODE_CHILDREN)
-            continue;
-
-        const auto& name = key.second;
-        const std::string normalized = normalizeClaimName(name, true);
-        if (normalized == key.second)
             continue;
 
+        auto& name = it.key();
         auto supports = getSupportsForName(name);
         for (auto support : supports) {
             // if it's already going to expire just skip it
@@ -279,23 +272,17 @@ std::vector<uint256> getClaimHashes(const CClaimTrieData& data)
 template <typename T>
 using iCbType = std::function<uint256(T&)>;
 
-template <typename T>
+template <typename TIterator>
-using decay = typename std::decay<T>::type;
+uint256 recursiveBinaryTreeHash(TIterator& it, const iCbType<TIterator>& process)
-
-template <typename Vector, typename T>
-uint256 recursiveMerkleHash(const CClaimTrieData& data, Vector&& children, const iCbType<T>& childHash)
 {
-    static_assert(std::is_same<decay<Vector>, std::vector<decay<T>>>::value, "Vector should be std vector");
-    static_assert(std::is_same<decltype(children[0]), T&>::value, "Vector element type should match callback type");
-
     std::vector<uint256> childHashes;
-    for (auto& child : children)
+    for (auto& child : it.children())
-        childHashes.emplace_back(childHash(child));
+        childHashes.emplace_back(process(child));
 
     std::vector<uint256> claimHashes;
-    if (!data.empty())
+    if (!it->empty())
-        claimHashes = getClaimHashes(data);
+        claimHashes = getClaimHashes(it.data());
-    else if (children.empty())
+    else if (!it.hasChildren())
         return {};
 
     auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes);
@@ -304,53 +291,44 @@ uint256 recursiveMerkleHash(const CClaimTrieData& data, Vector&& children, const
     return Hash(left.begin(), left.end(), right.begin(), right.end());
 }
 
-extern const uint256 one;
+uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(CClaimTrie::iterator& it)
-
-bool CClaimTrieHashFork::checkConsistency(const uint256& rootHash) const
-{
-    if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
-        return CClaimTrie::checkConsistency(rootHash);
-
-    CClaimTrieDataNode node;
-    if (!find({}, node) || node.hash != rootHash) {
-        if (rootHash == one)
-            return true;
-
-        return error("Mismatched root claim trie hashes. This may happen when there is not a clean process shutdown. Please run with -reindex.");
-    }
-
-    bool success = true;
-    recurseNodes({}, node, [&success, this](const std::string& name, const CClaimTrieData& data, const std::vector<std::string>& children) {
-        if (!success) return;
-
-        iCbType<const std::string> callback = [&success, &name, this](const std::string& child) -> uint256 {
-            auto key = name + child;
-            CClaimTrieDataNode node;
-            success &= find(key, node);
-            return node.hash;
-        };
-
-        success &= !data.hash.IsNull();
-        success &= data.hash == recursiveMerkleHash(data, children, callback);
-    });
-
-    return success;
-}
-
-uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it)
 {
     if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
         return CClaimTrieCacheNormalizationFork::recursiveComputeMerkleHash(it);
 
-    using iterator = CClaimPrefixTrie::iterator;
+    using iterator = CClaimTrie::iterator;
     iCbType<iterator> process = [&process](iterator& it) -> uint256 {
         if (it->hash.IsNull())
-            it->hash = recursiveMerkleHash(it.data(), it.children(), process);
+            it->hash = recursiveBinaryTreeHash(it, process);
+        assert(!it->hash.IsNull());
         return it->hash;
     };
     return process(it);
 }
 
+bool CClaimTrieCacheHashFork::recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const
+{
+    if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
+        return CClaimTrieCacheNormalizationFork::recursiveCheckConsistency(it, failed);
+
+    struct CRecursiveBreak {};
+    using iterator = CClaimTrie::const_iterator;
+    iCbType<iterator> process = [&failed, &process](iterator& it) -> uint256 {
+        if (it->hash.IsNull() || it->hash != recursiveBinaryTreeHash(it, process)) {
+            failed = it.key();
+            throw CRecursiveBreak();
+        }
+        return it->hash;
+    };
+
+    try {
+        process(it);
+    } catch (const CRecursiveBreak&) {
+        return false;
+    }
+    return true;
+}
+
 std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
 {
     uint32_t count = 0;
@@ -426,7 +404,7 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTri
     cacheData(name, false);
     getMerkleHash();
     proof = CClaimTrieProof();
-    for (auto& it : static_cast<const CClaimPrefixTrie&>(nodesToAddOrUpdate).nodes(name)) {
+    for (auto& it : static_cast<const CClaimTrie&>(nodesToAddOrUpdate).nodes(name)) {
         std::vector<uint256> childHashes;
         uint32_t nextCurrentIdx = 0;
         for (auto& child : it.children()) {
@@ -469,15 +447,12 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTri
 
 void CClaimTrieCacheHashFork::copyAllBaseToCache()
 {
-    recurseNodes({}, [this](const std::string& name, const CClaimTrieData& data) {
+    for (auto it = base->cbegin(); it != base->cend(); ++it)
-        if (nodesAlreadyCached.insert(name).second)
+        if (nodesAlreadyCached.insert(it.key()).second)
-            nodesToAddOrUpdate.insert(name, data);
+            nodesToAddOrUpdate.insert(it.key(), it.data());
-    });
 
-    for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) {
+    for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it)
         it->hash.SetNull();
-        it->flags |= CClaimTrieDataFlags::HASH_DIRTY;
-    }
 }
 
 void CClaimTrieCacheHashFork::initializeIncrement()
@@ -500,6 +475,7 @@ bool CClaimTrieCacheHashFork::finalizeDecrement(std::vector<std::pair<std::strin
     return ret;
 }
 
-bool CClaimTrieCacheHashFork::allowSupportMetadata() const {
+bool CClaimTrieCacheHashFork::allowSupportMetadata() const
+{
     return nNextHeight >= Params().GetConsensus().nAllClaimsInMerkleForkHeight;
 }

src/init.cpp

@@ -22,6 +22,7 @@
 #include <httprpc.h>
 #include <index/txindex.h>
 #include <key.h>
+#include <lbry.h>
 #include <validation.h>
 #include <miner.h>
 #include <netbase.h>
@@ -398,6 +399,7 @@ void SetupServerArgs()
     hidden_args.emplace_back("-sysperms");
 #endif
     gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-memfile=<GiB>", "Use a memory mapped file for the claimtrie allocations (default: use RAM instead)", false, OptionsCategory::OPTIONS);
 
     gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
@@ -1441,6 +1443,8 @@ bool AppInitMain()
     LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
     LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
 
+    g_memfileSize = gArgs.GetArg("-memfile", 0u);
+
     bool fLoaded = false;
     while (!fLoaded && !ShutdownRequested()) {
         bool fReset = fReindex;
@@ -1465,7 +1469,7 @@ bool AppInitMain()
                 int64_t trieCacheMB = gArgs.GetArg("-claimtriecache", nDefaultDbCache);
                 trieCacheMB = std::min(trieCacheMB, nMaxDbCache);
                 trieCacheMB = std::max(trieCacheMB, nMinDbCache);
-                pclaimTrie = new CClaimTrieHashFork(false, fReindex || fReindexChainState, 32, trieCacheMB);
+                pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState, 32, trieCacheMB);
 
                 if (fReset) {
                     pblocktree->WriteReindexing(true);

src/lbry.cpp

@@ -3,6 +3,8 @@
 
 #include <cstdio>
 
+uint32_t g_memfileSize = 0;
+
 unsigned int CalculateLbryNextWorkRequired(const CBlockIndex* pindexLast, int64_t nFirstBlockTime, const Consensus::Params& params)
 {
     if (params.fPowNoRetargeting)

src/lbry.h

@@ -4,6 +4,7 @@
 #include <chain.h>
 #include <chainparams.h>
 
+extern uint32_t g_memfileSize;
 unsigned int CalculateLbryNextWorkRequired(const CBlockIndex* pindexLast, int64_t nLastRetargetTime, const Consensus::Params& params);
 
 #endif

src/miner.cpp

@@ -208,7 +208,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc
     CValidationState state;
     if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) {
         if (!trieCache.empty())
-            trieCache.dumpToLog(trieCache.begin());
+            trieCache.dumpToLog(trieCache.find({}));
         throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state)));
     }
     int64_t nTime2 = GetTimeMicros();

src/prefixtrie.cpp

@@ -1,6 +1,78 @@
 
-#include "prefixtrie.h"
+#include <claimtrie.h>
-#include "claimtrie.h"
+#include <fs.h>
+#include <lbry.h>
+#include <limits>
+#include <memory>
+#include <prefixtrie.h>
+
+#include <boost/interprocess/allocators/private_node_allocator.hpp>
+#include <boost/interprocess/indexes/null_index.hpp>
+#include <boost/interprocess/managed_mapped_file.hpp>
+
+namespace bip = boost::interprocess;
+
+typedef bip::basic_managed_mapped_file <
+    char,
+    bip::rbtree_best_fit<bip::null_mutex_family, bip::offset_ptr<void>>,
+    bip::null_index
+> managed_mapped_file;
+
+template <typename T>
+using node_allocator = bip::private_node_allocator<T, managed_mapped_file::segment_manager>;
+
+static managed_mapped_file::segment_manager* segmentManager()
+{
+    struct CSharedMemoryFile
+    {
+        CSharedMemoryFile() : file(GetDataDir() / "shared.mem")
+        {
+            std::remove(file.c_str());
+            auto size = (uint64_t)g_memfileSize * 1024ULL * 1024ULL * 1024ULL;
+            menaged_file.reset(new managed_mapped_file(bip::create_only, file.c_str(), size));
+        }
+        ~CSharedMemoryFile()
+        {
+            menaged_file.reset();
+            std::remove(file.c_str());
+        }
+        managed_mapped_file::segment_manager* segmentManager()
+        {
+            return menaged_file->get_segment_manager();
+        }
+        const fs::path file;
+        std::unique_ptr<managed_mapped_file> menaged_file;
+    };
+    static CSharedMemoryFile shem;
+    return shem.segmentManager();
+}
+
+template <typename T>
+static node_allocator<T>& nodeAllocator()
+{
+    static node_allocator<T> allocator(segmentManager());
+    return allocator;
+}
+
+template <typename T, class... Args>
+static std::shared_ptr<T> nodeAllocate(Args&&... args)
+{
+    return std::allocate_shared<T>(nodeAllocator<T>(), std::forward<Args>(args)...);
+}
+
+template <typename T, class... Args>
+static std::shared_ptr<T> allocateShared(Args&&... args)
+{
+    static auto allocate = g_memfileSize ? nodeAllocate<T, Args...> : std::make_shared<T, Args...>;
+    try {
+        return allocate(std::forward<Args>(args)...);
+    }
+    catch (const bip::bad_alloc&) {
+        allocate = std::make_shared<T, Args...>; // in case we fill up the memfile
+        LogPrint(BCLog::BENCH, "WARNING: The memfile is full; reverting to the RAM allocator for %s.\n", typeid(T).name());
+        return allocate(std::forward<Args>(args)...);
+    }
+}
 
 template <typename TKey, typename TData>
 template <bool IsConst>
@@ -18,7 +90,7 @@ typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>& CPrefixTrie<TKey,
     stack.clear();
     stack.reserve(o.stack.size());
     for (auto& i : o.stack)
-        stack.push_back(Bookmark{i.name, i.parent, i.it, i.end});
+        stack.push_back(Bookmark{i.name, i.it, i.end});
     return *this;
 }
 
@@ -48,7 +120,7 @@ typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>& CPrefixTrie<TKey,
     if (!shared->children.empty()) {
         auto& children = shared->children;
         auto it = children.begin();
-        stack.emplace_back(Bookmark{name, shared, it, children.end()});
+        stack.emplace_back(Bookmark{name, it, children.end()});
         auto& postfix = it->first;
         name.insert(name.end(), postfix.begin(), postfix.end());
         node = it->second;
@@ -106,18 +178,30 @@ bool CPrefixTrie<TKey, TData>::Iterator<IsConst>::operator!=(const Iterator& o)
 
 template <typename TKey, typename TData>
 template <bool IsConst>
-typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>::reference CPrefixTrie<TKey, TData>::Iterator<IsConst>::operator*() const
+typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>::reference CPrefixTrie<TKey, TData>::Iterator<IsConst>::operator*()
 {
-    assert(!node.expired());
+    return reference{name, data()};
-    return TPair(name, node.lock()->data);
 }
 
 template <typename TKey, typename TData>
 template <bool IsConst>
-typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>::pointer CPrefixTrie<TKey, TData>::Iterator<IsConst>::operator->() const
+typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>::const_reference CPrefixTrie<TKey, TData>::Iterator<IsConst>::operator*() const
 {
-    assert(!node.expired());
+    return const_reference{name, data()};
-    return &(node.lock()->data);
+}
+
+template <typename TKey, typename TData>
+template <bool IsConst>
+typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>::pointer CPrefixTrie<TKey, TData>::Iterator<IsConst>::operator->()
+{
+    return &(data());
+}
+
+template <typename TKey, typename TData>
+template <bool IsConst>
+typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>::const_pointer CPrefixTrie<TKey, TData>::Iterator<IsConst>::operator->() const
+{
+    return &(data());
 }
 
 template <typename TKey, typename TData>
@@ -129,18 +213,20 @@ const TKey& CPrefixTrie<TKey, TData>::Iterator<IsConst>::key() const
 
 template <typename TKey, typename TData>
 template <bool IsConst>
-TData& CPrefixTrie<TKey, TData>::Iterator<IsConst>::data()
+typename CPrefixTrie<TKey, TData>::template Iterator<IsConst>::data_reference CPrefixTrie<TKey, TData>::Iterator<IsConst>::data()
 {
-    assert(!node.expired());
+    auto shared = node.lock();
-    return node.lock()->data;
+    assert(shared);
+    return *(shared->data);
 }
 
 template <typename TKey, typename TData>
 template <bool IsConst>
 const TData& CPrefixTrie<TKey, TData>::Iterator<IsConst>::data() const
 {
-    assert(!node.expired());
+    auto shared = node.lock();
-    return node.lock()->data;
+    assert(shared);
+    return *(shared->data);
 }
 
 template <typename TKey, typename TData>
@@ -240,19 +326,20 @@ std::shared_ptr<typename CPrefixTrie<TKey, TData>::Node>& CPrefixTrie<TKey, TDat
     }
     if (count == 0) {
         ++size;
-        it = children.emplace(key, std::make_shared<Node>()).first;
+        it = children.emplace(key, allocateShared<Node>()).first;
         return it->second;
     }
     if (count < it->first.size()) {
-        const TKey prefix(key.begin(), key.begin() + count);
+        TKey prefix(key.begin(), key.begin() + count);
-        const TKey postfix(it->first.begin() + count, it->first.end());
+        TKey postfix(it->first.begin() + count, it->first.end());
         auto nodes = std::move(it->second);
         children.erase(it);
         ++size;
-        it = children.emplace(prefix, std::make_shared<Node>()).first;
+        it = children.emplace(std::move(prefix), allocateShared<Node>()).first;
-        it->second->children.emplace(postfix, std::move(nodes));
+        it->second->children.emplace(std::move(postfix), std::move(nodes));
         if (key.size() == count)
             return it->second;
+        it->second->data = allocateShared<TData>();
     }
     return insert(TKey(key.begin() + count, key.end()), it->second);
 }
@@ -269,12 +356,12 @@ void CPrefixTrie<TKey, TData>::erase(const TKey& key, std::shared_ptr<Node>& nod
     if (!find(key, node, cb))
         return;
 
-    nodes.back().second->data = {};
+    nodes.back().second->data = allocateShared<TData>();
     for (; nodes.size() > 1; nodes.pop_back()) {
         // if we have only one child and no data ourselves, bring them up to our level
         auto& cNode = nodes.back().second;
         auto onlyOneChild = cNode->children.size() == 1;
-        auto noData = cNode->data.empty();
+        auto noData = cNode->data->empty();
         if (onlyOneChild && noData) {
             auto child = cNode->children.begin();
             auto& prefix = nodes.back().first;
@@ -282,7 +369,7 @@ void CPrefixTrie<TKey, TData>::erase(const TKey& key, std::shared_ptr<Node>& nod
             auto& postfix = child->first;
             newKey.insert(newKey.end(), postfix.begin(), postfix.end());
             auto& pNode = nodes[nodes.size() - 2].second;
-            pNode->children.emplace(newKey, std::move(child->second));
+            pNode->children.emplace(std::move(newKey), std::move(child->second));
             pNode->children.erase(prefix);
             --size;
             continue;
@@ -300,8 +387,9 @@ void CPrefixTrie<TKey, TData>::erase(const TKey& key, std::shared_ptr<Node>& nod
 }
 
 template <typename TKey, typename TData>
-CPrefixTrie<TKey, TData>::CPrefixTrie() : size(0), root(std::make_shared<Node>())
+CPrefixTrie<TKey, TData>::CPrefixTrie() : size(0), root(allocateShared<Node>())
 {
+    root->data = allocateShared<TData>();
 }
 
 template <typename TKey, typename TData>
@@ -309,7 +397,7 @@ template <typename TDataUni>
 typename CPrefixTrie<TKey, TData>::iterator CPrefixTrie<TKey, TData>::insert(const TKey& key, TDataUni&& data)
 {
     auto& node = key.empty() ? root : insert(key, root);
-    node->data = std::forward<TDataUni>(data);
+    node->data = allocateShared<TData>(std::forward<TDataUni>(data));
     return key.empty() ? begin() : iterator{key, node};
 }
 
@@ -333,9 +421,9 @@ typename CPrefixTrie<TKey, TData>::iterator CPrefixTrie<TKey, TData>::insert(CPr
         auto name = it.key();
         name.insert(name.end(), key.begin(), key.end());
         auto& node = insert(key, shared);
-        copy = iterator{name, node};
+        copy = iterator{std::move(name), node};
     }
-    copy.node.lock()->data = std::forward<TDataUni>(data);
+    copy.node.lock()->data = allocateShared<TData>(std::forward<TDataUni>(data));
     return copy;
 }
 
@@ -404,7 +492,7 @@ bool CPrefixTrie<TKey, TData>::erase(const TKey& key)
 {
     auto size_was = height();
     if (key.empty()) {
-        root->data = {};
+        root->data = allocateShared<TData>();
     } else {
         erase(key, root);
     }
@@ -415,7 +503,7 @@ template <typename TKey, typename TData>
 void CPrefixTrie<TKey, TData>::clear()
 {
     size = 0;
-    root->data = {};
+    root->data = allocateShared<TData>();
     root->children.clear();
 }
 
@@ -428,7 +516,7 @@ bool CPrefixTrie<TKey, TData>::empty() const
 template <typename TKey, typename TData>
 std::size_t CPrefixTrie<TKey, TData>::height() const
 {
-    return size + (root->data.empty() ? 0 : 1);
+    return size + (root->data->empty() ? 0 : 1);
 }
 
 template <typename TKey, typename TData>

src/prefixtrie.h

@@ -27,9 +27,9 @@ class CPrefixTrie
         Node() = default;
         Node(const Node&) = delete;
         Node(Node&& o) noexcept = default;
-        Node& operator=(Node&& o) noexcept = default;
+        Node& operator=(Node&&) noexcept = default;
         Node& operator=(const Node&) = delete;
-        TData data;
+        std::shared_ptr<TData> data;
     };
 
     using TChildren = decltype(Node::children);
@@ -42,15 +42,15 @@ class CPrefixTrie
         friend class CPrefixTrie<TKey, TData>;
 
         using TKeyRef = std::reference_wrapper<const TKey>;
-        using TDataRef = std::reference_wrapper<TData>;
+        using TDataRef = std::reference_wrapper<typename std::conditional<IsConst, const TData, TData>::type>;
         using TPair = std::pair<TKeyRef, TDataRef>;
+        using ConstTPair = std::pair<TKeyRef, const TData>;
 
         TKey name;
         std::weak_ptr<Node> node;
 
         struct Bookmark {
             TKey name;
-            std::weak_ptr<Node> parent;
             typename TChildren::iterator it;
             typename TChildren::iterator end;
         };
@@ -60,8 +60,11 @@ class CPrefixTrie
     public:
         // Iterator traits
         using value_type = TPair;
+        using const_pointer = const TData* const;
+        using const_reference = ConstTPair;
+        using data_reference = typename std::conditional<IsConst, const TData&, TData&>::type;
         using pointer = typename std::conditional<IsConst, const TData* const, TData* const>::type;
-        using reference = typename std::conditional<IsConst, const TPair, TPair>::type;
+        using reference = typename std::conditional<IsConst, ConstTPair, TPair>::type;
         using difference_type = std::ptrdiff_t;
         using iterator_category = std::forward_iterator_tag;
 
@@ -89,12 +92,15 @@ class CPrefixTrie
         bool operator==(const Iterator& o) const;
         bool operator!=(const Iterator& o) const;
 
-        reference operator*() const;
+        reference operator*();
-        pointer operator->() const;
+        const_reference operator*() const;
+
+        pointer operator->();
+        const_pointer operator->() const;
 
         const TKey& key() const;
 
-        TData& data();
+        data_reference data();
         const TData& data() const;
 
         std::size_t depth() const;

src/rpc/claimtrie.cpp

@@ -307,7 +307,7 @@ static UniValue getclaimsintrie(const JSONRPCRequest& request)
     }
 
     UniValue ret(UniValue::VARR);
-    trieCache.recurseNodes({}, [&ret, &trieCache, &coinsCache] (const std::string& name, const CClaimTrieData& data) {
+    trieCache.iterate([&ret, &trieCache, &coinsCache] (const std::string& name, const CClaimTrieData& data) {
         if (ShutdownRequested())
             throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested");
 
@@ -348,7 +348,7 @@ static UniValue getnamesintrie(const JSONRPCRequest& request)
     }
 
     UniValue ret(UniValue::VARR);
-    trieCache.recurseNodes({}, [&ret](const std::string &name, const CClaimTrieData &data) {
+    trieCache.iterate([&ret](const std::string &name, const CClaimTrieData &data) {
         if (!data.empty())
             ret.push_back(escapeNonUtf8(name));
         if (ShutdownRequested())

src/test/claimtriebranching_tests.cpp

@@ -18,7 +18,7 @@ BOOST_AUTO_TEST_CASE(claim_replace_test) {
     fixture.Spend(tx1);
     CMutableTransaction tx2 = fixture.MakeClaim(fixture.GetCoinbase(), "bassfisher", "one", 1);
     fixture.IncrementBlocks(1);
-    BOOST_CHECK(pclaimTrie->checkConsistency(fixture.getMerkleHash()));
+    BOOST_CHECK(fixture.checkConsistency());
     BOOST_CHECK(!fixture.is_best_claim("bass", tx1));
     BOOST_CHECK(fixture.is_best_claim("bassfisher", tx2));
 }
@@ -1852,17 +1852,18 @@ BOOST_AUTO_TEST_CASE(update_on_support2_test)
     CMutableTransaction s2 = fixture.MakeSupport(fixture.GetCoinbase(), tx1, name, 1);
     fixture.IncrementBlocks(1);
 
-    CClaimTrieData node;
+    auto it = pclaimTrie->find(name);
-    BOOST_CHECK(pclaimTrie->find(name, node));
+    BOOST_CHECK(it);
-    BOOST_CHECK_EQUAL(node.nHeightOfLastTakeover, height + 1);
+    BOOST_CHECK_EQUAL(it->nHeightOfLastTakeover, height + 1);
 
     fixture.Spend(s1);
     fixture.Spend(s2);
     CMutableTransaction u1 = fixture.MakeUpdate(tx1, name, value, ClaimIdHash(tx1.GetHash(), 0), 3);
     fixture.IncrementBlocks(1);
 
-    BOOST_CHECK(pclaimTrie->find(name, node));
+    it = pclaimTrie->find(name);
-    BOOST_CHECK_EQUAL(node.nHeightOfLastTakeover, height + 1);
+    BOOST_CHECK(it);
+    BOOST_CHECK_EQUAL(it->nHeightOfLastTakeover, height + 1);
 }
 
 BOOST_AUTO_TEST_SUITE_END()

src/test/claimtriecache_tests.cpp

@@ -37,7 +37,7 @@ public:
         return nodesToAddOrUpdate.height();
     }
 
-    CClaimPrefixTrie::iterator getCache(const std::string& key)
+    CClaimTrie::iterator getCache(const std::string& key)
     {
         return nodesToAddOrUpdate.find(key);
     }
@@ -108,7 +108,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
 
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
-    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
+    BOOST_CHECK(ntState.checkConsistency());
 
     CClaimTrieCacheTest ntState1(pclaimTrie);
     ntState1.removeClaimFromTrie(std::string("test"), tx1OutPoint, unused, true);
@@ -128,7 +128,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
 
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3);
-    BOOST_CHECK(pclaimTrie->checkConsistency(hash3));
+    BOOST_CHECK(ntState2.checkConsistency());
 
     CClaimTrieCacheTest ntState3(pclaimTrie);
     ntState3.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, hash160, 50, 100, 200), true);
@@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState3.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4);
-    BOOST_CHECK(pclaimTrie->checkConsistency(hash4));
+    BOOST_CHECK(ntState3.checkConsistency());
 
     CClaimTrieCacheTest ntState4(pclaimTrie);
     ntState4.removeClaimFromTrie(std::string("abab"), tx6OutPoint, unused, true);
@@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState4.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
-    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
+    BOOST_CHECK(ntState4.checkConsistency());
 
     CClaimTrieCacheTest ntState5(pclaimTrie);
     ntState5.removeClaimFromTrie(std::string("test"), tx3OutPoint, unused, true);
@@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState5.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
-    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
+    BOOST_CHECK(ntState5.checkConsistency());
 
     CClaimTrieCacheTest ntState6(pclaimTrie);
     ntState6.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, hash160, 50, 101, 201), true);
@@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState6.flush();
     BOOST_CHECK(!pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
-    BOOST_CHECK(pclaimTrie->checkConsistency(hash2));
+    BOOST_CHECK(ntState6.checkConsistency());
 
     CClaimTrieCacheTest ntState7(pclaimTrie);
     ntState7.removeClaimFromTrie(std::string("test"), tx3OutPoint, unused, true);
@@ -174,7 +174,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
     ntState7.flush();
     BOOST_CHECK(pclaimTrie->empty());
     BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
-    BOOST_CHECK(pclaimTrie->checkConsistency(hash0));
+    BOOST_CHECK(ntState7.checkConsistency());
 }
 
 BOOST_AUTO_TEST_CASE(basic_insertion_info_test)
@@ -281,14 +281,12 @@ BOOST_AUTO_TEST_CASE(iteratetrie_test)
     ctc.insertClaimIntoTrie("test", claimVal, true);
     BOOST_CHECK(ctc.flush());
 
-    CClaimTrieDataNode node;
+    auto hit = pclaimTrie->find("");
-    BOOST_CHECK(pclaimTrie->find("", node));
+    BOOST_CHECK(hit);
-    BOOST_CHECK_EQUAL(node.children.size(), 1U);
+    BOOST_CHECK_EQUAL(hit.children().size(), 1U);
-    BOOST_CHECK(pclaimTrie->find("test", node));
+    BOOST_CHECK(hit = pclaimTrie->find("test"));
-    BOOST_CHECK_EQUAL(node.children.size(), 0U);
+    BOOST_CHECK_EQUAL(hit.children().size(), 0U);
-    CClaimTrieData data;
+    BOOST_CHECK_EQUAL(hit.data().claims.size(), 1);
-    BOOST_CHECK(pclaimTrie->find("test", data));
-    BOOST_CHECK_EQUAL(data.claims.size(), 1);
 }
 
 BOOST_AUTO_TEST_CASE(trie_stays_consistent_test)
@@ -305,13 +303,13 @@ BOOST_AUTO_TEST_CASE(trie_stays_consistent_test)
         BOOST_CHECK(cache.insertClaimIntoTrie(name, value, false));
 
     cache.flush();
-    BOOST_CHECK(trie.checkConsistency(cache.getMerkleHash()));
+    BOOST_CHECK(cache.checkConsistency());
 
     for (auto& name: names) {
         CClaimValue temp;
         BOOST_CHECK(cache.removeClaimFromTrie(name, COutPoint(), temp, false));
         cache.flush();
-        BOOST_CHECK(trie.checkConsistency(cache.getMerkleHash()));
+        BOOST_CHECK(cache.checkConsistency());
     }
     BOOST_CHECK(trie.empty());
 }
@@ -384,7 +382,6 @@ BOOST_AUTO_TEST_CASE(verify_basic_serialization)
     ssData >> cv2;
 
     BOOST_CHECK_EQUAL(cv, cv2);
-    BOOST_CHECK_EQUAL(cv.nEffectiveAmount, cv2.nEffectiveAmount);
 }
 
 BOOST_AUTO_TEST_CASE(claimtrienode_serialize_unserialize)

src/test/claimtrienormalization_tests.cpp

@@ -93,8 +93,7 @@ BOOST_AUTO_TEST_CASE(claimtriebranching_normalization)
     BOOST_CHECK(fixture.is_best_claim("normalizetest", tx1));
     BOOST_CHECK(fixture.best_claim_effective_amount_equals("normalizetest", 3));
 
-    CClaimTrieData data;
+    BOOST_CHECK(!pclaimTrie->find("normalizeTest"));
-    BOOST_CHECK(!pclaimTrie->find("normalizeTest", data));
 
     // Check equivalence of normalized claim names
     BOOST_CHECK(fixture.is_best_claim("normalizetest", tx1)); // collapsed tx2
@@ -223,8 +222,7 @@ BOOST_AUTO_TEST_CASE(claimtriecache_normalization)
     BOOST_CHECK(!trieCache.spendClaim(name_normd, COutPoint(tx2.GetHash(), 0), currentHeight, amelieValidHeight));
     BOOST_CHECK(trieCache.spendClaim(name_upper, COutPoint(tx2.GetHash(), 0), currentHeight, amelieValidHeight));
 
-    CClaimTrieData data;
+    BOOST_CHECK(!pclaimTrie->find(name));
-    BOOST_CHECK(!pclaimTrie->find(name, data));
     BOOST_CHECK(trieCache.getInfoForName(name, nval1));
     BOOST_CHECK(trieCache.addClaim(name, COutPoint(tx1.GetHash(), 0), ClaimIdHash(tx1.GetHash(), 0), CAmount(2), currentHeight + 1));
     BOOST_CHECK(trieCache.getInfoForName(name, nval1));
@@ -236,8 +234,9 @@ BOOST_AUTO_TEST_CASE(claimtriecache_normalization)
     BOOST_CHECK(trieCache.incrementBlock(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo, takeoverHeightUndo));
     BOOST_CHECK(trieCache.shouldNormalize());
 
-    CClaimTrieDataNode node;
+    // we cannot use getXXXForName cause they will normalized name
-    BOOST_CHECK(!pclaimTrie->find(name, node));
+    for (auto it = pclaimTrie->cbegin(); it != pclaimTrie->cend(); ++it)
+        BOOST_CHECK(it.key() != name);
 }
 
 BOOST_AUTO_TEST_CASE(undo_normalization_does_not_kill_claim_order)

src/test/test_bitcoin.cpp

@@ -144,7 +144,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha
         pblocktree.reset(new CBlockTreeDB(1 << 20, true));
         pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true));
         pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get()));
-        pclaimTrie = new CClaimTrieHashFork(true, false, 1);
+        pclaimTrie = new CClaimTrie(true, false, 1);
         if (!LoadGenesisBlock(chainparams)) {
             throw std::runtime_error("LoadGenesisBlock failed.");
         }

src/validation.cpp

@@ -1562,12 +1562,8 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI
     assert(trieCache.finalizeDecrement(blockUndo.takeoverHeightUndo));
     auto merkleHash = trieCache.getMerkleHash();
     if (merkleHash != pindex->pprev->hashClaimTrie) {
-        for (auto cit = trieCache.begin(); cit != trieCache.end(); ++cit) {
+        if (!trieCache.empty())
-            if (cit->claims.size() && cit->nHeightOfLastTakeover <= 0)
+            trieCache.dumpToLog(trieCache.find({}));
-                LogPrintf("Invalid takeover height discovered in cache for %s\n", cit.key());
-            if (cit->hash.IsNull())
-                LogPrintf("Invalid hash discovered in cache for %s\n", cit.key());
-        }
         LogPrintf("Hash comparison failure at block %d\n", pindex->nHeight);
         assert(merkleHash == pindex->pprev->hashClaimTrie);
     }
@@ -2045,7 +2041,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
     if (trieCache.getMerkleHash() != block.hashClaimTrie)
     {
         if (!trieCache.empty()) // we could run checkConsistency here, but it would take a while
-            trieCache.dumpToLog(trieCache.begin());
+            trieCache.dumpToLog(trieCache.find({}));
         return state.DoS(100, error("ConnectBlock() : the merkle root of the claim trie does not match "
                                "(actual=%s vs block=%s on height=%d)", trieCache.getMerkleHash().GetHex(),
                                block.hashClaimTrie.GetHex(), pindex->nHeight), REJECT_INVALID, "bad-claim-merkle-hash");
@@ -2188,7 +2184,7 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
             // overwrite one. Still, use a conservative safety factor of 2.
             if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize()))
                 return state.Error("out of disk space");
-            if (!pclaimTrie->SyncToDisk())
+            if (mode == FlushStateMode::ALWAYS && !pclaimTrie->SyncToDisk())
                 return state.Error("Failed to write to claim trie database");
             // Flush the chainstate (which may refer to block index entries).
             if (!pcoinsTip->Flush())