From 6b8935718e99103ed3a2bba763d35829b79ddffa Mon Sep 17 00:00:00 2001 From: Brannon King Date: Thu, 1 Aug 2019 18:35:58 -0600 Subject: [PATCH] first pass at not loading full claimtrie into RAM tweaks --- src/claimtrie.cpp | 403 +++++++++++++++----------- src/claimtrie.h | 71 +++-- src/claimtrieforks.cpp | 37 ++- src/init.cpp | 2 +- src/miner.cpp | 2 +- src/prefixtrie.cpp | 15 - src/prefixtrie.h | 15 + src/rpc/claimtrie.cpp | 84 +++--- src/serialize.h | 14 +- src/test/claimtriebranching_tests.cpp | 27 +- src/test/claimtriecache_tests.cpp | 76 ++--- src/validation.cpp | 18 +- 12 files changed, 436 insertions(+), 328 deletions(-) diff --git a/src/claimtrie.cpp b/src/claimtrie.cpp index d075dbc06..8a0dccdb6 100644 --- a/src/claimtrie.cpp +++ b/src/claimtrie.cpp @@ -8,8 +8,6 @@ #include #include -#include - static const uint256 one = uint256S("0000000000000000000000000000000000000000000000000000000000000001"); std::vector heightToVch(int n) @@ -123,13 +121,13 @@ void CClaimTrieData::reorderClaims(const supportEntryType& supports) claim.nEffectiveAmount += support.nAmount; } - std::make_heap(claims.begin(), claims.end()); + std::sort(claims.rbegin(), claims.rend()); } CClaimTrie::CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor) { nProportionalDelayFactor = proportionalDelayFactor; - db.reset(new CDBWrapper(GetDataDir() / "claimtrie", 100 * 1024 * 1024, fMemory, fWipe, false)); + db.reset(new CDBWrapper(GetDataDir() / "claimtrie", 200 * 1024 * 1024, fMemory, fWipe, false)); } bool CClaimTrie::SyncToDisk() @@ -200,7 +198,7 @@ typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow(co template <> typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueueCacheRow(int nHeight, bool createIfNotExists) { - return getQueue(*(base->db), EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists); + return getQueue(*(base->db), CLAIM_EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists); } template <> @@ -218,8 +216,13 @@ typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueu bool CClaimTrieCacheBase::haveClaim(const std::string& name, const COutPoint& outPoint) const { - auto it = find(name); - return it && it->haveClaim(outPoint); + auto it = nodesToAddOrUpdate.find(name); + if (it && it->haveClaim(outPoint)) + return true; + if (it || nodesToDelete.count(name)) + return false; + CClaimTrieDataNode node; + return base->find(name, node) && node.data.haveClaim(outPoint); } bool CClaimTrieCacheBase::haveSupport(const std::string& name, const COutPoint& outPoint) const @@ -272,39 +275,62 @@ bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COut return haveInQueue(name, outPoint, nValidAtHeight); } -std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const +void CClaimTrie::recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function function) const { + function(name, current); + for (auto& child: current.children) { + CClaimTrieDataNode node; + if (find(child.second, node)) + recurseAllHashedNodes(name + child.first, node, function); + } +} + +std::size_t CClaimTrie::getTotalNamesInTrie() const { std::size_t count = 0; - for (auto it = base->cbegin(); it != base->cend(); ++it) - if (!it->empty()) ++count; + CClaimTrieDataNode node; + if (find("", node)) + recurseAllHashedNodes("", node, [&count](const std::string&, const CClaimTrieDataNode& node) { + count += !node.data.empty(); + }); return count; } -std::size_t CClaimTrieCacheBase::getTotalClaimsInTrie() const +std::size_t CClaimTrie::getTotalClaimsInTrie() const { std::size_t count = 0; - for (auto it = base->cbegin(); it != base->cend(); ++it) - count += it->claims.size(); + CClaimTrieDataNode node; + if (find("", node)) + recurseAllHashedNodes("", node, [&count](const std::string&, const CClaimTrieDataNode& node) { + count += node.data.claims.size(); + }); return count; } -CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) const +CAmount CClaimTrie::getTotalValueOfClaimsInTrie(bool fControllingOnly) const { CAmount value_in_subtrie = 0; - for (auto it = base->cbegin(); it != base->cend(); ++it) { - for (const auto& claim : it->claims) { - value_in_subtrie += claim.nAmount; - if (fControllingOnly) - break; - } - } + std::size_t count = 0; + CClaimTrieDataNode node; + if (find("", node)) + recurseAllHashedNodes("", node, [&value_in_subtrie, fControllingOnly](const std::string&, const CClaimTrieDataNode& node) { + for (const auto& claim : node.data.claims) { + value_in_subtrie += claim.nAmount; + if (fControllingOnly) + break; + } + }); return value_in_subtrie; } bool CClaimTrieCacheBase::getInfoForName(const std::string& name, CClaimValue& claim) const { - auto it = find(name); - return it && it->getBestClaim(claim); + auto it = nodesToAddOrUpdate.find(name); + if (it && it->getBestClaim(claim)) + return true; + if (it || nodesToDelete.count(name)) + return false; + CClaimTrieDataNode node; + return base->find(name, node) && node.data.getBestClaim(claim); } CClaimsForNameType CClaimTrieCacheBase::getClaimsForName(const std::string& name) const @@ -313,10 +339,15 @@ CClaimsForNameType CClaimTrieCacheBase::getClaimsForName(const std::string& name int nLastTakeoverHeight = 0; auto supports = getSupportsForName(name); - if (auto it = find(name)) { + CClaimTrieDataNode node; + if (auto it = nodesToAddOrUpdate.find(name)) { claims = it->claims; nLastTakeoverHeight = it->nHeightOfLastTakeover; } + else if (!nodesToDelete.count(name) && base->find(name, node)) { + claims = node.data.claims; + nLastTakeoverHeight = node.data.nHeightOfLastTakeover; + } return {std::move(claims), std::move(supports), nLastTakeoverHeight, name}; } @@ -381,60 +412,92 @@ uint256 recursiveMerkleHash(TIterator& it, const iCbType& process, co return Hash(vchToHash.begin(), vchToHash.end()); } -bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) +bool CClaimTrie::checkConsistency(const uint256& rootHash) const { - struct CRecursiveBreak : public std::exception {}; + CClaimTrieDataNode node; + if (!find("", node) || node.data.hash != rootHash) { + if (rootHash == one) + return true; - using iterator = CClaimTrie::const_iterator; - iCbType verify = [&failed](iterator& it) { - if (!it.hasChildren()) { - // we don't allow a situation of no children and no claims; no empty leaf nodes allowed - failed = it.key(); - throw CRecursiveBreak(); - } - }; - - iCbType process = [&failed, &process, &verify](iterator& it) { - if (it->hash != recursiveMerkleHash(it, process, verify)) { - failed = it.key(); - throw CRecursiveBreak(); - } - }; - - try { - process(it); - } catch (const CRecursiveBreak&) { - return false; + return error("Mismatched root claim trie hashes. This may happen when there is not a clean process shutdown. Please run with -reindex."); } - return true; + + bool success = true; + recurseAllHashedNodes("", node, [&success](const std::string& name, const CClaimTrieDataNode& node) { + std::vector vchToHash; + const auto pos = name.size(); + for (auto& child : node.children) { + auto key = name + child.first; + auto hash = child.second; + completeHash(hash, key, pos); + vchToHash.push_back(key[pos]); + vchToHash.insert(vchToHash.end(), hash.begin(), hash.end()); + } + + CClaimValue claim; + if (node.data.getBestClaim(claim)) { + uint256 valueHash = getValueHash(claim.outPoint, node.data.nHeightOfLastTakeover); + vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end()); + } else { + success &= !node.children.empty(); // we disallow leaf nodes without claims + } + + success &= node.data.hash == Hash(vchToHash.begin(), vchToHash.end()); + }); + + return success; } -bool CClaimTrieCacheBase::checkConsistency() const -{ - if (base->empty()) - return true; +std::vector> CClaimTrie::nodes(const std::string &key) const { + std::vector> ret; + CClaimTrieDataNode node; - auto it = base->cbegin(); - std::string failed; - auto consistent = recursiveCheckConsistency(it, failed); - if (!consistent) { - LogPrintf("\nPrinting base tree from its parent:\n"); - auto basePath = base->nodes(failed); - if (basePath.size() > 1) basePath.pop_back(); - dumpToLog(basePath.back(), false); - auto cachePath = nodesToAddOrUpdate.nodes(failed); - if (!cachePath.empty()) { - LogPrintf("\nPrinting %s's parent from cache:\n", failed); - if (cachePath.size() > 1) cachePath.pop_back(); - dumpToLog(cachePath.back(), false); - } - if (!nodesToDelete.empty()) { - std::string joined; - for (const auto &piece : nodesToDelete) joined += ", " + piece; - LogPrintf("Nodes to be deleted: %s\n", joined.substr(2)); + if (!find("", node)) + return ret; + ret.emplace_back("", node); + + std::string partialKey = key; + + while (!node.children.empty()) { + auto it = node.children.lower_bound(partialKey); + if (it != node.children.end() && it->first == partialKey) { + // we're completely done + if (find(it->second, node)) + ret.emplace_back(key, node); + break; } + if (it != node.children.begin()) --it; + const auto count = match(partialKey, it->first); + + if (count != it->first.size()) break; + if (count == partialKey.size()) break; + partialKey = partialKey.substr(count); + if (find(it->second, node)) + ret.emplace_back(key.substr(0, key.size() - partialKey.size()), node); + else break; } - return consistent; + + return ret; +} + +bool CClaimTrie::contains(const std::string &key) const { + return db->Exists(std::make_pair(TRIE_NODE_BY_NAME, key)); +} + +bool CClaimTrie::empty() const { + return !contains(""); +} + +bool CClaimTrie::find(const std::string &key, CClaimTrieDataNode &node) const { + uint256 hash; + if (!db->Read(std::make_pair(TRIE_NODE_BY_NAME, key), hash)) + return false; + auto found = find(hash, node); + return found; +} + +bool CClaimTrie::find(const uint256 &key, CClaimTrieDataNode &node) const { + return db->Read(std::make_pair(TRIE_NODE_BY_HASH, key), node); } bool CClaimTrieCacheBase::getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const @@ -486,99 +549,79 @@ bool CClaimTrieCacheBase::flush() getMerkleHash(); + std::set forDeletion; for (const auto& nodeName : nodesToDelete) { - if (nodesToAddOrUpdate.contains(nodeName)) - continue; + // TODO: we don't need to deserialize all the nodes right here + // we could be smarter about this and fill in the whole list in removeClaimFromTrie auto nodes = base->nodes(nodeName); - base->erase(nodeName); for (auto& node : nodes) - if (!node) - batch.Erase(std::make_pair(TRIE_NODE, node.key())); + forDeletion.insert(node.first); } for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) { - auto old = base->find(it.key()); - if (!old || old.data() != it.data()) { - base->copy(it); - batch.Write(std::make_pair(TRIE_NODE, it.key()), it.data()); - } + forDeletion.erase(it.key()); + if (!dirtyNodes.count(it.key())) + continue; + + CClaimTrieDataNode node; + node.data = it.data(); + for (auto &child: it.children()) + node.children.emplace(child.key().substr(it.key().size()), child->hash); + + batch.Write(std::make_pair(TRIE_NODE_BY_HASH, it->hash), node); + batch.Write(std::make_pair(TRIE_NODE_BY_NAME, it.key()), it->hash); + } + + for (auto& name: forDeletion) { + batch.Erase(std::make_pair(TRIE_NODE_BY_NAME, name)); } BatchWriteQueue(batch, SUPPORT, supportCache); + BatchWriteQueue(batch, CLAIM_QUEUE_ROW, claimQueueCache); BatchWriteQueue(batch, CLAIM_QUEUE_NAME_ROW, claimQueueNameCache); - BatchWriteQueue(batch, EXP_QUEUE_ROW, expirationQueueCache); + BatchWriteQueue(batch, CLAIM_EXP_QUEUE_ROW, expirationQueueCache); + BatchWriteQueue(batch, SUPPORT_QUEUE_ROW, supportQueueCache); BatchWriteQueue(batch, SUPPORT_QUEUE_NAME_ROW, supportQueueNameCache); BatchWriteQueue(batch, SUPPORT_EXP_QUEUE_ROW, supportExpirationQueueCache); base->nNextHeight = nNextHeight; - if (!nodesToAddOrUpdate.empty()) - LogPrint(BCLog::CLAIMS, "Cache size: %zu from base size: %zu on block %d\n", nodesToAddOrUpdate.height(), base->height(), nNextHeight); + if (!nodesToAddOrUpdate.empty() && (LogAcceptCategory(BCLog::CLAIMS) || LogAcceptCategory(BCLog::BENCH))) { + LogPrintf("TrieCache size: %zu nodes on block %d, batch writes %zu bytes.\n", + nodesToAddOrUpdate.height(), nNextHeight, batch.SizeEstimate()); + } auto ret = base->db->WriteBatch(batch); clear(); return ret; } -bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip) +bool CClaimTrieCacheBase::validateTrieConsistency(const CBlockIndex* tip) { - LogPrintf("Loading the claim trie from disk...\n"); - - base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0; - - clear(); - base->clear(); - boost::scoped_ptr pcursor(base->db->NewIterator()); - - std::vector> hashesOnEmptyNodes; - - for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) { - std::pair key; - if (!pcursor->GetKey(key) || key.first != TRIE_NODE) - continue; - - CClaimTrieData data; - if (pcursor->GetValue(data)) { - if (data.empty()) { - // we have a situation where our old trie had many empty nodes - // we don't want to automatically throw those all into our prefix trie - hashesOnEmptyNodes.emplace_back(key.second, data.hash); - continue; - } - - // nEffectiveAmount isn't serialized but it needs to be initialized (as done in reorderClaims): - auto supports = getSupportsForName(key.second); - data.reorderClaims(supports); - base->insert(key.second, std::move(data)); - } else { - return error("%s(): error reading claim trie from disk", __func__); - } - } - - CDBBatch batch(*(base->db)); - for (auto& kvp: hashesOnEmptyNodes) { - auto hit = base->find(kvp.first); - if (hit != base->end()) - hit->hash = kvp.second; - else { - // the first time the prefix trie is ran there will be many unused nodes - // we need to clean those out so that we can go faster next time - batch.Erase(std::make_pair(TRIE_NODE, kvp.first)); - } - } + if (!tip || tip->nHeight < 1) + return true; LogPrintf("Checking claim trie consistency... "); - if (checkConsistency()) { + if (base->checkConsistency(tip->hashClaimTrie)) { LogPrintf("consistent\n"); - if (tip && tip->hashClaimTrie != getMerkleHash()) - return error("%s(): hashes don't match when reading claimtrie from disk", __func__); - base->db->WriteBatch(batch); return true; } LogPrintf("inconsistent!\n"); return false; } +bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip) +{ + base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0; + clear(); + + if (tip && (base->db->Exists(std::make_pair(TRIE_NODE, std::string())) || !base->db->Exists(std::make_pair(TRIE_NODE_BY_HASH, tip->hashClaimTrie)))) { + LogPrintf("The claim trie database contains deprecated data and will need to be rebuilt"); + return false; + } + return validateTrieConsistency(tip); +} + CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) : base(base) { assert(base); @@ -590,9 +633,9 @@ int CClaimTrieCacheBase::expirationTime() const return Params().GetConsensus().nOriginalClaimExpirationTime; } -uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimTrie::iterator& it) +uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it) { - using iterator = CClaimTrie::iterator; + using iterator = CClaimPrefixTrie::iterator; iCbType process = [&process](iterator& it) { if (it->hash.IsNull()) it->hash = recursiveMerkleHash(it, process); @@ -604,54 +647,51 @@ uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimTrie::iterator& it uint256 CClaimTrieCacheBase::getMerkleHash() { auto it = nodesToAddOrUpdate.begin(); - if (nodesToAddOrUpdate.empty() && nodesToDelete.empty()) - it = base->begin(); - return !it ? one : recursiveComputeMerkleHash(it); + if (it) + return recursiveComputeMerkleHash(it); + if (nodesToDelete.empty() && nodesAlreadyCached.empty()) { + CClaimTrieDataNode node; + if (base->find("", node)) + return node.data.hash; // it may be valuable to have base cache its current root hash + } + return one; // we have no data or we deleted everything } -CClaimTrie::const_iterator CClaimTrieCacheBase::begin() const +CClaimPrefixTrie::const_iterator CClaimTrieCacheBase::begin() const { - return nodesToAddOrUpdate.empty() && nodesToDelete.empty() ? base->cbegin() : nodesToAddOrUpdate.begin(); + return nodesToAddOrUpdate.begin(); } -CClaimTrie::const_iterator CClaimTrieCacheBase::end() const +CClaimPrefixTrie::const_iterator CClaimTrieCacheBase::end() const { - return nodesToAddOrUpdate.empty() && nodesToDelete.empty() ? base->cend() : nodesToAddOrUpdate.end(); -} - -CClaimTrie::const_iterator CClaimTrieCacheBase::find(const std::string& name) const -{ - if (auto it = nodesToAddOrUpdate.find(name)) - return it; - return base->find(name); + return nodesToAddOrUpdate.end(); } bool CClaimTrieCacheBase::empty() const { - return base->empty() && nodesToAddOrUpdate.empty(); + return nodesToAddOrUpdate.empty(); } -CClaimTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& name, bool create) +CClaimPrefixTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& name, bool create) { - // get data from the cache. if no data, create empty one - const auto insert = [this](CClaimTrie::iterator& it) { - auto& key = it.key(); - // we only ever cache nodes once per cache instance - if (!nodesAlreadyCached.count(key)) { - // do not insert nodes that are already present - nodesAlreadyCached.insert(key); - nodesToAddOrUpdate.insert(key, it.data()); - } - }; - // we need all parent nodes and their one level deep children // to calculate merkle hash auto nodes = base->nodes(name); for (auto& node: nodes) { - for (auto& child : node.children()) - if (!nodesAlreadyCached.count(child.key())) - nodesToAddOrUpdate.copy(child); - insert(node); + if (!nodesAlreadyCached.count(node.first)) { + // do not insert nodes that are already present + nodesAlreadyCached.insert(node.first); + nodesToAddOrUpdate.insert(node.first, node.second.data); + } + for (auto& child : node.second.children) { + auto childKey = node.first + child.first; + if (!nodesAlreadyCached.count(childKey)) { + CClaimTrieDataNode childNode; + if (base->find(child.second, childNode)) { + nodesToAddOrUpdate.insert(childKey, childNode.data); + } + } + } } auto it = nodesToAddOrUpdate.find(name); @@ -677,10 +717,11 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16 std::tie(claimId, takeoverHeight) = cit->second; return true; } - if (auto it = base->find(name)) { - takeoverHeight = it->nHeightOfLastTakeover; + CClaimTrieDataNode data; + if (base->find(name, data)) { + takeoverHeight = data.data.nHeightOfLastTakeover; CClaimValue claim; - if (it->getBestClaim(claim)) { + if (data.data.getBestClaim(claim)) { claimId = claim.claimId; return true; } @@ -690,8 +731,10 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16 void CClaimTrieCacheBase::markAsDirty(const std::string& name, bool fCheckTakeover) { - for (auto& node : nodesToAddOrUpdate.nodes(name)) + for (auto& node : nodesToAddOrUpdate.nodes(name)) { + dirtyNodes.insert(node.key()); node->hash.SetNull(); + } if (fCheckTakeover) namesToCheckForTakeover.insert(name); @@ -712,7 +755,7 @@ bool CClaimTrieCacheBase::removeClaimFromTrie(const std::string& name, const COu auto it = cacheData(name, false); if (!it || !it->removeClaim(outPoint, claim)) { - LogPrint(BCLog::CLAIMS, "%s: Removing a claim was unsuccessful. name = %s, txhash = %s, nOut = %d", __func__, name, outPoint.hash.GetHex(), outPoint.n); + LogPrint(BCLog::CLAIMS, "%s: Removing a claim was unsuccessful. name = %s, txhash = %s, nOut = %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n); return false; } @@ -963,11 +1006,13 @@ bool CClaimTrieCacheBase::removeSupportFromMap(const std::string& name, const CO return false; } -void CClaimTrieCacheBase::dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase) const +void CClaimTrieCacheBase::dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase) const { + if (!it) return; + if (diffFromBase) { - auto hit = base->find(it.key()); - if (hit && hit->hash == it->hash) + CClaimTrieDataNode node; + if (base->find(it.key(), node) && node.data.hash == it->hash) return; } @@ -1281,8 +1326,15 @@ int CClaimTrieCacheBase::getNumBlocksOfContinuousOwnership(const std::string& na that->removalWorkaround.erase(hit); return 0; } - auto it = find(name); - return it && !it->empty() ? nNextHeight - it->nHeightOfLastTakeover : 0; + auto it = nodesToAddOrUpdate.find(name); + if (it && !it->empty()) + return nNextHeight - it->nHeightOfLastTakeover; + if (it) // we specifically ignore deleted nodes here to allow this to fall into the base lookup in that scenario + return 0; + CClaimTrieDataNode node; + if (base->find(name, node) && !node.data.empty()) + return nNextHeight - node.data.nHeightOfLastTakeover; + return 0; } int CClaimTrieCacheBase::getDelayForName(const std::string& name) const @@ -1311,6 +1363,7 @@ bool CClaimTrieCacheBase::clear() { nodesToAddOrUpdate.clear(); claimsToAddToByIdIndex.clear(); + dirtyNodes.clear(); supportCache.clear(); nodesToDelete.clear(); claimsToDeleteFromByIdIndex.clear(); diff --git a/src/claimtrie.h b/src/claimtrie.h index 46c4e8088..973bbca96 100644 --- a/src/claimtrie.h +++ b/src/claimtrie.h @@ -18,11 +18,13 @@ #include // leveldb keys -#define TRIE_NODE 'n' +#define TRIE_NODE 'n' // deprecated +#define TRIE_NODE_BY_HASH 'h' +#define TRIE_NODE_BY_NAME 'g' #define CLAIM_BY_ID 'i' #define CLAIM_QUEUE_ROW 'r' #define CLAIM_QUEUE_NAME_ROW 'm' -#define EXP_QUEUE_ROW 'e' +#define CLAIM_EXP_QUEUE_ROW 'e' #define SUPPORT 's' #define SUPPORT_QUEUE_ROW 'u' #define SUPPORT_QUEUE_NAME_ROW 'p' @@ -61,6 +63,7 @@ struct CClaimValue READWRITE(nAmount); READWRITE(nHeight); READWRITE(nValidAtHeight); + READWRITE(nEffectiveAmount); } bool operator<(const CClaimValue& other) const @@ -188,6 +191,26 @@ struct CClaimTrieData } }; +struct CClaimTrieDataNode { + CClaimTrieData data; + std::map children; + + CClaimTrieDataNode() = default; + CClaimTrieDataNode(CClaimTrieDataNode&&) = default; + CClaimTrieDataNode(const CClaimTrieDataNode&) = default; + CClaimTrieDataNode& operator=(CClaimTrieDataNode&&) = default; + CClaimTrieDataNode& operator=(const CClaimTrieDataNode& d) = default; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(children); + READWRITE(data); // keep item using "s.eof" last + } +}; + struct COutPointHeightType { COutPoint outPoint; @@ -301,7 +324,7 @@ struct CClaimsForNameType CClaimsForNameType& operator=(const CClaimsForNameType&) = default; }; -class CClaimTrie : public CPrefixTrie +class CClaimTrie { int nNextHeight = 0; int nProportionalDelayFactor = 0; @@ -322,6 +345,19 @@ public: friend struct ClaimTrieChainFixture; friend class CClaimTrieCacheExpirationFork; friend class CClaimTrieCacheNormalizationFork; + + std::size_t getTotalNamesInTrie() const; + std::size_t getTotalClaimsInTrie() const; + CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const; + bool checkConsistency(const uint256& rootHash) const; + + bool contains(const std::string& key) const; + bool empty() const; + bool find(const uint256& key, CClaimTrieDataNode& node) const; + bool find(const std::string& key, CClaimTrieDataNode& node) const; + + std::vector> nodes(const std::string& key) const; + void recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function function) const; }; struct CClaimTrieProofNode @@ -381,6 +417,8 @@ typedef std::map expirationQueueType; typedef std::set claimIndexClaimListType; typedef std::vector claimIndexElementListType; +typedef CPrefixTrie CClaimPrefixTrie; + class CClaimTrieCacheBase { public: @@ -388,7 +426,6 @@ public: virtual ~CClaimTrieCacheBase() = default; uint256 getMerkleHash(); - bool checkConsistency() const; bool getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const; @@ -402,10 +439,6 @@ public: bool haveSupport(const std::string& name, const COutPoint& outPoint) const; bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight); - std::size_t getTotalNamesInTrie() const; - std::size_t getTotalClaimsInTrie() const; - CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const; - bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight); bool undoAddClaim(const std::string& name, const COutPoint& outPoint, int nHeight); @@ -441,18 +474,18 @@ public: CAmount getEffectiveAmountForClaim(const std::string& name, const uint160& claimId, std::vector* supports = nullptr) const; CAmount getEffectiveAmountForClaim(const CClaimsForNameType& claims, const uint160& claimId, std::vector* supports = nullptr) const; - CClaimTrie::const_iterator begin() const; - CClaimTrie::const_iterator end() const; - CClaimTrie::const_iterator find(const std::string& name) const; + CClaimPrefixTrie::const_iterator begin() const; + CClaimPrefixTrie::const_iterator end() const; - void dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase = true) const; + void dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase = true) const; + virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const; protected: CClaimTrie* base; - CClaimTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush + CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush std::unordered_set namesToCheckForTakeover; // takeover numbers are updated on increment - uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it); + uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it); virtual bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover); virtual bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover); @@ -460,14 +493,12 @@ protected: virtual bool insertSupportIntoMap(const std::string& name, const CSupportValue& support, bool fCheckTakeover); virtual bool removeSupportFromMap(const std::string& name, const COutPoint& outPoint, CSupportValue& support, bool fCheckTakeover); - virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const; - supportEntryType getSupportsForName(const std::string& name) const; int getDelayForName(const std::string& name) const; virtual int getDelayForName(const std::string& name, const uint160& claimId) const; - CClaimTrie::iterator cacheData(const std::string& name, bool create = true); + CClaimPrefixTrie::iterator cacheData(const std::string& name, bool create = true); bool getLastTakeoverForName(const std::string& name, uint160& claimId, int& takeoverHeight) const; @@ -499,6 +530,7 @@ private: std::unordered_set nodesAlreadyCached; // set of nodes already pulled into cache from base std::unordered_map takeoverWorkaround; std::unordered_set removalWorkaround; + std::unordered_set dirtyNodes; bool shouldUseTakeoverWorkaround(const std::string& key) const; void addTakeoverWorkaroundPotential(const std::string& key); @@ -510,6 +542,8 @@ private: bool removeSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover); bool removeClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover); + bool validateTrieConsistency(const CBlockIndex* tip); + template std::pair>>* getQueueCacheRow(int nHeight, bool createIfNotExists = false); @@ -614,6 +648,7 @@ public: bool getProofForName(const std::string& name, CClaimTrieProof& proof) override; bool getInfoForName(const std::string& name, CClaimValue& claim) const override; CClaimsForNameType getClaimsForName(const std::string& name) const override; + std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override; protected: bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover) override; @@ -624,8 +659,6 @@ protected: int getDelayForName(const std::string& name, const uint160& claimId) const override; - std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override; - private: bool overrideInsertNormalization; bool overrideRemoveNormalization; diff --git a/src/claimtrieforks.cpp b/src/claimtrieforks.cpp index 7f4983c25..ab823c546 100644 --- a/src/claimtrieforks.cpp +++ b/src/claimtrieforks.cpp @@ -8,6 +8,7 @@ #include #include #include +#include CClaimTrieCacheExpirationFork::CClaimTrieCacheExpirationFork(CClaimTrie* base) : CClaimTrieCacheBase(base) @@ -66,7 +67,7 @@ bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment) if (!pcursor->GetKey(key)) continue; int height = key.second; - if (key.first == EXP_QUEUE_ROW) { + if (key.first == CLAIM_EXP_QUEUE_ROW) { expirationQueueRowType row; if (pcursor->GetValue(row)) { reactivateClaim(row, height, increment); @@ -160,40 +161,48 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insert // run the one-time upgrade of all names that need to change // it modifies the (cache) trie as it goes, so we need to grab everything to be modified first - for (auto it = base->begin(); it != base->end(); ++it) { - const std::string normalized = normalizeClaimName(it.key(), true); - if (normalized == it.key()) + boost::scoped_ptr pcursor(base->db->NewIterator()); + for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) { + std::pair key; + if (!pcursor->GetKey(key) || key.first != TRIE_NODE_BY_NAME) continue; - auto supports = getSupportsForName(it.key()); + const auto& name = key.second; + const std::string normalized = normalizeClaimName(name, true); + if (normalized == key.second) + continue; + + auto supports = getSupportsForName(name); for (auto support : supports) { // if it's already going to expire just skip it if (support.nHeight + expirationTime() <= nNextHeight) continue; - assert(removeSupportFromMap(it.key(), support.outPoint, support, false)); - expireSupportUndo.emplace_back(it.key(), support); + assert(removeSupportFromMap(name, support.outPoint, support, false)); + expireSupportUndo.emplace_back(name, support); assert(insertSupportIntoMap(normalized, support, false)); - insertSupportUndo.emplace_back(it.key(), support.outPoint, -1); + insertSupportUndo.emplace_back(name, support.outPoint, -1); } namesToCheckForTakeover.insert(normalized); - auto cached = cacheData(it.key(), false); + auto cached = cacheData(name, false); if (!cached || cached->empty()) continue; - for (auto claim : it->claims) { + auto claimsCopy = cached->claims; + auto takeoverHeightCopy = cached->nHeightOfLastTakeover; + for (auto claim : claimsCopy) { if (claim.nHeight + expirationTime() <= nNextHeight) continue; - assert(removeClaimFromTrie(it.key(), claim.outPoint, claim, false)); - removeUndo.emplace_back(it.key(), claim); + assert(removeClaimFromTrie(name, claim.outPoint, claim, false)); + removeUndo.emplace_back(name, claim); assert(insertClaimIntoTrie(normalized, claim, true)); - insertUndo.emplace_back(it.key(), claim.outPoint, -1); + insertUndo.emplace_back(name, claim.outPoint, -1); } - takeoverHeightUndo.emplace_back(it.key(), it->nHeightOfLastTakeover); + takeoverHeightUndo.emplace_back(name, takeoverHeightCopy); } return true; } diff --git a/src/init.cpp b/src/init.cpp index 26408bd79..679cbfff8 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1461,7 +1461,7 @@ bool AppInitMain() pblocktree.reset(); pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); delete pclaimTrie; - pclaimTrie = new CClaimTrie(false, fReindex); + pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState); if (fReset) { pblocktree->WriteReindexing(true); diff --git a/src/miner.cpp b/src/miner.cpp index dcfd3c6b8..9a07b3978 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -207,7 +207,7 @@ std::unique_ptr BlockAssembler::CreateNewBlock(const CScript& sc CValidationState state; if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { - if (!trieCache.empty() && trieCache.checkConsistency()) + if (!trieCache.empty()) trieCache.dumpToLog(trieCache.begin()); throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); } diff --git a/src/prefixtrie.cpp b/src/prefixtrie.cpp index b000ef1be..1dc832b1e 100644 --- a/src/prefixtrie.cpp +++ b/src/prefixtrie.cpp @@ -174,21 +174,6 @@ std::vector::template Iterator> CPref return ret; } -template -static std::size_t match(const TKey& a, const TKey& b) -{ - std::size_t count = 0; - auto ait = a.cbegin(), aend = a.cend(); - auto bit = b.cbegin(), bend = b.cend(); - while (ait != aend && bit != bend) { - if (*ait != *bit) break; - ++count; - ++ait; - ++bit; - } - return count; -} - template template TIterator CPrefixTrie::find(const TKey& key, TNode node, TIterator end) diff --git a/src/prefixtrie.h b/src/prefixtrie.h index 99ad0576d..d821bd283 100644 --- a/src/prefixtrie.h +++ b/src/prefixtrie.h @@ -193,4 +193,19 @@ inline bool operator!=(const std::reference_wrapper>& ref, co return !(ref == obj); } +template +static std::size_t match(const TKey& a, const TKey& b) +{ + std::size_t count = 0; + auto ait = a.cbegin(), aend = a.cend(); + auto bit = b.cbegin(), bend = b.cend(); + while (ait != aend && bit != bend) { + if (*ait != *bit) break; + ++count; + ++ait; + ++bit; + } + return count; +} + #endif // BITCOIN_PREFIXTRIE_H diff --git a/src/rpc/claimtrie.cpp b/src/rpc/claimtrie.cpp index 9a4e13c5d..4202056bd 100644 --- a/src/rpc/claimtrie.cpp +++ b/src/rpc/claimtrie.cpp @@ -174,48 +174,52 @@ static UniValue getclaimsintrie(const JSONRPCRequest& request) throw JSONRPCError(RPC_METHOD_DEPRECATED, msg); } + UniValue ret(UniValue::VARR); + uint256 rootHash; LOCK(cs_main); CCoinsViewCache coinsCache(pcoinsTip.get()); CClaimTrieCache trieCache(pclaimTrie); if (!request.params.empty()) { - CBlockIndex* blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)")); + CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)")); RollBackTo(blockIndex, coinsCache, trieCache); } + rootHash = trieCache.getMerkleHash(); - UniValue ret(UniValue::VARR); - for (auto it = trieCache.begin(); it != trieCache.end(); ++it) - { + CClaimTrieDataNode rootNode; + if (!pclaimTrie->find(rootHash, rootNode)) + return ret; + + pclaimTrie->recurseAllHashedNodes("", rootNode, [&ret, &trieCache, &coinsCache](const std::string &name, + const CClaimTrieDataNode &node) { if (ShutdownRequested()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested"); boost::this_thread::interruption_point(); - if (it->empty()) - continue; + if (node.data.empty()) + return; UniValue claims(UniValue::VARR); - for (auto itClaims = it->claims.cbegin(); itClaims != it->claims.cend(); ++itClaims) { + for (auto itClaims = node.data.claims.cbegin(); itClaims != node.data.claims.cend(); ++itClaims) { UniValue claim(UniValue::VOBJ); claim.pushKV("claimId", itClaims->claimId.GetHex()); claim.pushKV("txid", itClaims->outPoint.hash.GetHex()); - claim.pushKV("n", (int)itClaims->outPoint.n); + claim.pushKV("n", (int) itClaims->outPoint.n); claim.pushKV("amount", ValueFromAmount(itClaims->nAmount)); claim.pushKV("height", itClaims->nHeight); - const Coin& coin = coinsCache.AccessCoin(itClaims->outPoint); - if (coin.IsSpent()) - { - LogPrintf("%s: the specified txout of %s appears to have been spent\n", __func__, itClaims->outPoint.hash.GetHex()); + const Coin &coin = coinsCache.AccessCoin(itClaims->outPoint); + if (coin.IsSpent()) { + LogPrintf("%s: the specified txout of %s appears to have been spent\n", __func__, + itClaims->outPoint.hash.GetHex()); claim.pushKV("error", "Txout spent"); - } - else - { + } else { int op; std::vector > vvchParams; - if (!DecodeClaimScript(coin.out.scriptPubKey, op, vvchParams)) - { - LogPrintf("%s: the specified txout of %s does not have an claim command\n", __func__, itClaims->outPoint.hash.GetHex()); + if (!DecodeClaimScript(coin.out.scriptPubKey, op, vvchParams)) { + LogPrintf("%s: the specified txout of %s does not have an claim command\n", __func__, + itClaims->outPoint.hash.GetHex()); } claim.pushKV("value", HexStr(vvchParams[1].begin(), vvchParams[1].end())); } @@ -228,11 +232,10 @@ static UniValue getclaimsintrie(const JSONRPCRequest& request) } UniValue nodeObj(UniValue::VOBJ); - nodeObj.pushKV("normalized_name", escapeNonUtf8(it.key())); + nodeObj.pushKV("normalized_name", escapeNonUtf8(name)); nodeObj.pushKV("claims", claims); ret.push_back(nodeObj); - } - + }); return ret; } @@ -258,27 +261,33 @@ static UniValue getnamesintrie(const JSONRPCRequest& request) "Result: \n" "\"names\" (array) all names in the trie that have claims\n"); - LOCK(cs_main); + uint256 rootHash; + { + LOCK(cs_main); - CCoinsViewCache coinsCache(pcoinsTip.get()); - CClaimTrieCache trieCache(pclaimTrie); + CCoinsViewCache coinsCache(pcoinsTip.get()); + CClaimTrieCache trieCache(pclaimTrie); - if (!request.params.empty()) { - CBlockIndex* blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)")); - RollBackTo(blockIndex, coinsCache, trieCache); + if (!request.params.empty()) { + CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)")); + RollBackTo(blockIndex, coinsCache, trieCache); + } + rootHash = trieCache.getMerkleHash(); } - UniValue ret(UniValue::VARR); - for (auto it = trieCache.begin(); it != trieCache.end(); ++it) { + CClaimTrieDataNode rootNode; + if (!pclaimTrie->find(rootHash, rootNode)) + return ret; + + pclaimTrie->recurseAllHashedNodes("", rootNode, [&ret](const std::string& name, const CClaimTrieDataNode& node) { + if (!node.data.empty()) + ret.push_back(escapeNonUtf8(name)); if (ShutdownRequested()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested"); boost::this_thread::interruption_point(); - - if (!it->empty()) - ret.push_back(escapeNonUtf8(it.key())); - } + }); return ret; } @@ -580,8 +589,7 @@ UniValue gettotalclaimednames(const JSONRPCRequest& request) " names in the trie\n" ); LOCK(cs_main); - CClaimTrieCache trieCache(pclaimTrie); - auto num_names = trieCache.getTotalNamesInTrie(); + auto num_names = pclaimTrie->getTotalNamesInTrie(); return int(num_names); } @@ -597,8 +605,7 @@ UniValue gettotalclaims(const JSONRPCRequest& request) " of active claims\n" ); LOCK(cs_main); - CClaimTrieCache trieCache(pclaimTrie); - auto num_claims = trieCache.getTotalClaimsInTrie(); + auto num_claims = pclaimTrie->getTotalClaimsInTrie(); return int(num_claims); } @@ -619,8 +626,7 @@ UniValue gettotalvalueofclaims(const JSONRPCRequest& request) bool controlling_only = false; if (request.params.size() == 1) controlling_only = request.params[0].get_bool(); - CClaimTrieCache trieCache(pclaimTrie); - auto total_amount = trieCache.getTotalValueOfClaimsInTrie(controlling_only); + auto total_amount = pclaimTrie->getTotalValueOfClaimsInTrie(controlling_only); return ValueFromAmount(total_amount); } diff --git a/src/serialize.h b/src/serialize.h index 8bd3ef5d7..f2ac76b10 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -559,8 +559,8 @@ template void Unserialize(Stream& is, s /** * map */ -template void Serialize(Stream& os, const std::map& m); -template void Unserialize(Stream& is, std::map& m); +template void Serialize(Stream& os, const std::map& m); +template void Unserialize(Stream& is, std::map& m); /** * set @@ -781,20 +781,20 @@ void Unserialize(Stream& is, std::pair& item) /** * map */ -template -void Serialize(Stream& os, const std::map& m) +template +void Serialize(Stream& os, const std::map& m) { WriteCompactSize(os, m.size()); for (const auto& entry : m) Serialize(os, entry); } -template -void Unserialize(Stream& is, std::map& m) +template +void Unserialize(Stream& is, std::map& m) { m.clear(); unsigned int nSize = ReadCompactSize(is); - typename std::map::iterator mi = m.begin(); + typename std::map::iterator mi = m.begin(); for (unsigned int i = 0; i < nSize; i++) { std::pair item; diff --git a/src/test/claimtriebranching_tests.cpp b/src/test/claimtriebranching_tests.cpp index d58601bbe..dd4709901 100644 --- a/src/test/claimtriebranching_tests.cpp +++ b/src/test/claimtriebranching_tests.cpp @@ -351,7 +351,7 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork if (!expirationQueue.second.empty()) return false; } - return keyTypeEmpty(EXP_QUEUE_ROW); + return keyTypeEmpty(CLAIM_EXP_QUEUE_ROW); } bool supportEmpty() @@ -431,6 +431,8 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork } } } + + std::size_t getTotalNamesInTrie() const { return base->getTotalNamesInTrie(); } }; BOOST_FIXTURE_TEST_SUITE(claimtriebranching_tests, RegTestingSetup) @@ -495,6 +497,8 @@ BOOST_AUTO_TEST_CASE(triehash_fuzzer_test) } } fixture.IncrementBlocks(1); + if (blocks > 13 && i % 50 == 0) // travisCI needs some periodic output + std::cerr << "In triehash_fuzzer_test with " << fixture.getTotalNamesInTrie() << " names at block " << i << std::endl; } if (blocks == 1000 && claimsPerBlock == 100) @@ -1021,7 +1025,7 @@ BOOST_AUTO_TEST_CASE(claimtrie_expire_test) BOOST_CHECK(fixture.is_best_claim("test", tx6)); } /* - * tests for CClaimTrie::getEffectiveAmountForClaim + * tests for CClaimPrefixTrie::getEffectiveAmountForClaim */ BOOST_AUTO_TEST_CASE(claimtriebranching_get_effective_amount_for_claim) { @@ -1066,7 +1070,7 @@ BOOST_AUTO_TEST_CASE(claimtriebranching_get_effective_amount_for_claim) } /* - * tests for CClaimTrie::getClaimById basic consistency checks + * tests for CClaimPrefixTrie::getClaimById basic consistency checks */ BOOST_AUTO_TEST_CASE(get_claim_by_id_test) { @@ -1544,10 +1548,8 @@ BOOST_AUTO_TEST_CASE(claimtriecache_normalization) BOOST_CHECK(trieCache.incrementBlock(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo, takeoverHeightUndo)); BOOST_CHECK(trieCache.shouldNormalize()); - // we cannot use getXXXForName cause they will normalized name - for (auto it = pclaimTrie->begin(); it != pclaimTrie->end(); ++it) { - BOOST_CHECK(it.key() != name); - } + CClaimTrieDataNode node; + BOOST_CHECK(!pclaimTrie->find(name, node)); } BOOST_AUTO_TEST_CASE(undo_normalization_does_not_kill_claim_order) @@ -2183,7 +2185,7 @@ BOOST_AUTO_TEST_CASE(insert_update_claim_test) << std::vector(sName1.begin(), sName1.end()) << std::vector(tx1ClaimId.begin(), tx1ClaimId.end()) << std::vector(sValue1.begin(), sValue1.end()) << OP_2DROP << OP_2DROP << OP_TRUE; - tx8.vout[0].nValue = tx8.vout[0].nValue - 1; + tx8.vout[0].nValue -= 1; tx8.vout[1].scriptPubKey = CScript() << OP_CLAIM_NAME << std::vector(sName1.begin(), sName1.end()) << std::vector(sValue2.begin(), sValue2.end()) << OP_2DROP << OP_DROP << OP_TRUE; @@ -3817,7 +3819,7 @@ BOOST_AUTO_TEST_CASE(bogus_claimtrie_hash_test) } /* - * tests for CClaimTrie::getClaimById basic consistency checks + * tests for CClaimPrefixTrie::getClaimById basic consistency checks */ BOOST_AUTO_TEST_CASE(get_claim_by_id_test_2) { @@ -4099,14 +4101,17 @@ BOOST_AUTO_TEST_CASE(update_on_support2_test) CMutableTransaction s2 = fixture.MakeSupport(fixture.GetCoinbase(), tx1, name, 1); fixture.IncrementBlocks(1); - BOOST_CHECK_EQUAL(pclaimTrie->find(name)->nHeightOfLastTakeover, height + 1); + CClaimTrieDataNode node; + BOOST_CHECK(pclaimTrie->find(name, node)); + BOOST_CHECK_EQUAL(node.data.nHeightOfLastTakeover, height + 1); fixture.Spend(s1); fixture.Spend(s2); CMutableTransaction u1 = fixture.MakeUpdate(tx1, name, value, ClaimIdHash(tx1.GetHash(), 0), 3); fixture.IncrementBlocks(1); - BOOST_CHECK_EQUAL(pclaimTrie->find(name)->nHeightOfLastTakeover, height + 1); + BOOST_CHECK(pclaimTrie->find(name, node)); + BOOST_CHECK_EQUAL(node.data.nHeightOfLastTakeover, height + 1); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/claimtriecache_tests.cpp b/src/test/claimtriecache_tests.cpp index 698370735..9613f4896 100644 --- a/src/test/claimtriecache_tests.cpp +++ b/src/test/claimtriecache_tests.cpp @@ -36,7 +36,7 @@ public: return nodesToAddOrUpdate.height(); } - CClaimTrie::iterator getCache(const std::string& key) + CClaimPrefixTrie::iterator getCache(const std::string& key) { return nodesToAddOrUpdate.find(key); } @@ -124,7 +124,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2); - BOOST_CHECK(ntState.checkConsistency()); + BOOST_CHECK(pclaimTrie->checkConsistency(hash2)); CClaimTrieCacheTest ntState1(pclaimTrie); ntState1.removeClaimFromTrie(std::string("test"), tx1OutPoint, unused, true); @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3); - BOOST_CHECK(ntState2.checkConsistency()); + BOOST_CHECK(pclaimTrie->checkConsistency(hash3)); CClaimTrieCacheTest ntState3(pclaimTrie); ntState3.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, hash160, 50, 100, 200), true); @@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) ntState3.flush(); BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4); - BOOST_CHECK(ntState3.checkConsistency()); + BOOST_CHECK(pclaimTrie->checkConsistency(hash4)); CClaimTrieCacheTest ntState4(pclaimTrie); ntState4.removeClaimFromTrie(std::string("abab"), tx6OutPoint, unused, true); @@ -160,7 +160,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) ntState4.flush(); BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2); - BOOST_CHECK(ntState4.checkConsistency()); + BOOST_CHECK(pclaimTrie->checkConsistency(hash2)); CClaimTrieCacheTest ntState5(pclaimTrie); ntState5.removeClaimFromTrie(std::string("test"), tx3OutPoint, unused, true); @@ -169,7 +169,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) ntState5.flush(); BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2); - BOOST_CHECK(ntState5.checkConsistency()); + BOOST_CHECK(pclaimTrie->checkConsistency(hash2)); CClaimTrieCacheTest ntState6(pclaimTrie); ntState6.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, hash160, 50, 101, 201), true); @@ -178,7 +178,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) ntState6.flush(); BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2); - BOOST_CHECK(ntState6.checkConsistency()); + BOOST_CHECK(pclaimTrie->checkConsistency(hash2)); CClaimTrieCacheTest ntState7(pclaimTrie); ntState7.removeClaimFromTrie(std::string("test"), tx3OutPoint, unused, true); @@ -190,7 +190,7 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) ntState7.flush(); BOOST_CHECK(pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0); - BOOST_CHECK(ntState7.checkConsistency()); + BOOST_CHECK(pclaimTrie->checkConsistency(hash0)); } BOOST_AUTO_TEST_CASE(basic_insertion_info_test) @@ -299,28 +299,12 @@ BOOST_AUTO_TEST_CASE(iteratetrie_test) BOOST_CHECK(ctc.flush()); std::size_t count = 0; - for (auto it = pclaimTrie->begin(); it != pclaimTrie->end(); ++it) { - ++count; - if (it.key() == "test") { - BOOST_CHECK_EQUAL(it->claims.size(), 1); - } - } - BOOST_CHECK_EQUAL(count, 2); - - count = 0; - for (const auto& it: *pclaimTrie) { - ++count; - if (it.first == "test") { - const CClaimTrieData& data = it.second; - BOOST_CHECK_EQUAL(data.claims.size(), 1); - } - } - BOOST_CHECK_EQUAL(count, 2); - - auto it = pclaimTrie->find("test"); - BOOST_CHECK(it != pclaimTrie->end()); - BOOST_CHECK_EQUAL(it->claims.size(), 1); - BOOST_CHECK_EQUAL(pclaimTrie->height(), 1); + CClaimTrieDataNode node; + BOOST_CHECK(pclaimTrie->find("", node)); + BOOST_CHECK_EQUAL(node.children.size(), 1U); + BOOST_CHECK(pclaimTrie->find("test", node)); + BOOST_CHECK_EQUAL(node.children.size(), 0U); + BOOST_CHECK_EQUAL(node.data.claims.size(), 1); } BOOST_AUTO_TEST_CASE(trie_stays_consistent_test) @@ -337,15 +321,15 @@ BOOST_AUTO_TEST_CASE(trie_stays_consistent_test) BOOST_CHECK(cache.insertClaimIntoTrie(name, value, false)); cache.flush(); - BOOST_CHECK(cache.checkConsistency()); + BOOST_CHECK(trie.checkConsistency(cache.getMerkleHash())); for (auto& name: names) { CClaimValue temp; BOOST_CHECK(cache.removeClaimFromTrie(name, COutPoint(), temp, false)); cache.flush(); - BOOST_CHECK(cache.checkConsistency()); + BOOST_CHECK(trie.checkConsistency(cache.getMerkleHash())); } - BOOST_CHECK_EQUAL(trie.height(), 0); + BOOST_CHECK(trie.empty()); } BOOST_AUTO_TEST_CASE(takeover_workaround_triggers) @@ -394,9 +378,29 @@ BOOST_AUTO_TEST_CASE(takeover_workaround_triggers) BOOST_CHECK(cache.incrementBlock(icu, ecu, isu, esu, thu)); - BOOST_CHECK_EQUAL(3, cache.find("aa")->nHeightOfLastTakeover); - BOOST_CHECK_EQUAL(3, cache.find("bb")->nHeightOfLastTakeover); - BOOST_CHECK_EQUAL(1, cache.find("cc")->nHeightOfLastTakeover); + BOOST_CHECK_EQUAL(3, cache.getCache("aa")->nHeightOfLastTakeover); + BOOST_CHECK_EQUAL(3, cache.getCache("bb")->nHeightOfLastTakeover); + BOOST_CHECK_EQUAL(1, cache.getCache("cc")->nHeightOfLastTakeover); +} + +BOOST_AUTO_TEST_CASE(verify_basic_serialization) +{ + CClaimValue cv; + cv.outPoint = COutPoint(uint256S("123"), 2); + cv.nHeight = 3; + cv.claimId.SetHex("4567"); + cv.nEffectiveAmount = 4; + cv.nAmount = 5; + cv.nValidAtHeight = 6; + + CDataStream ssData(SER_NETWORK, PROTOCOL_VERSION); + ssData << cv; + + CClaimValue cv2; + ssData >> cv2; + + BOOST_CHECK_EQUAL(cv, cv2); + BOOST_CHECK_EQUAL(cv.nEffectiveAmount, cv2.nEffectiveAmount); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/validation.cpp b/src/validation.cpp index 2a6b9c170..f141fc67c 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1562,13 +1562,11 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI assert(trieCache.finalizeDecrement(blockUndo.takeoverHeightUndo)); auto merkleHash = trieCache.getMerkleHash(); if (merkleHash != pindex->pprev->hashClaimTrie) { - if (trieCache.checkConsistency()) { - for (auto cit = trieCache.begin(); cit != trieCache.end(); ++cit) { - if (cit->claims.size() && cit->nHeightOfLastTakeover <= 0) - LogPrintf("Invalid takeover height discovered in cache for %s\n", cit.key()); - if (cit->hash.IsNull()) - LogPrintf("Invalid hash discovered in cache for %s\n", cit.key()); - } + for (auto cit = trieCache.begin(); cit != trieCache.end(); ++cit) { + if (cit->claims.size() && cit->nHeightOfLastTakeover <= 0) + LogPrintf("Invalid takeover height discovered in cache for %s\n", cit.key()); + if (cit->hash.IsNull()) + LogPrintf("Invalid hash discovered in cache for %s\n", cit.key()); } LogPrintf("Hash comparison failure at block %d\n", pindex->nHeight); assert(merkleHash == pindex->pprev->hashClaimTrie); @@ -2048,7 +2046,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl if (trieCache.getMerkleHash() != block.hashClaimTrie) { - if (!trieCache.empty() && trieCache.checkConsistency()) + if (!trieCache.empty()) // we could run checkConsistency here, but it would take a while trieCache.dumpToLog(trieCache.begin()); return state.DoS(100, error("ConnectBlock() : the merkle root of the claim trie does not match " "(actual=%s vs block=%s on height=%d)", trieCache.getMerkleHash().GetHex(), @@ -2296,13 +2294,13 @@ void static UpdateTip(const CBlockIndex *pindexNew, const CChainParams& chainPar auto oldBlock = pindexNew->nTime + MAX_FUTURE_BLOCK_TIME < currentTime; if (!warningMessages.empty() || !oldBlock || lastBlockPrintTime < currentTime - 15 || LogAcceptCategory(BCLog::CLAIMS)) { lastBlockPrintTime = currentTime; - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g txb=%lu tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo) trie nodes=%zu%s", + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g txb=%lu tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, log(pindexNew->nChainWork.getdouble()) / log(2.0), (unsigned long) pindexNew->nTx, (unsigned long) pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()), GuessVerificationProgress(chainParams.TxData(), pindexNew), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1U << 20U)), pcoinsTip->GetCacheSize(), - pclaimTrie->height(), oldBlock ? " (synchronizing)" : ""); + oldBlock ? " (synchronizing)" : ""); if (!warningMessages.empty()) LogPrintf(" warning='%s'", warningMessages); /* Continued */ LogPrintf("\n");