From 580f6e20eb0c117fa757db56002a8694de075407 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Fri, 19 Jul 2019 18:36:53 +0300 Subject: [PATCH] Implement binary tree hash algorithm Signed-off-by: Anthony Fieroni --- src/chainparams.cpp | 3 + src/claimtrie.cpp | 43 ++--- src/claimtrie.h | 59 ++++-- src/claimtrieforks.cpp | 261 +++++++++++++++++++++++++- src/consensus/params.h | 2 + src/init.cpp | 2 +- src/miner.cpp | 2 +- src/prefixtrie.cpp | 48 +++-- src/prefixtrie.h | 8 +- src/rpc/claimtrie.cpp | 69 ++++--- src/test/claimtriebranching_tests.cpp | 170 ++++++++++++++++- src/test/claimtriecache_tests.cpp | 1 - src/test/test_bitcoin.cpp | 2 +- src/validation.cpp | 6 +- 14 files changed, 565 insertions(+), 111 deletions(-) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 073000f72..bae77de99 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -146,6 +146,7 @@ public: consensus.nMinTakeoverWorkaroundHeight = 496850; consensus.nMaxTakeoverWorkaroundHeight = 10000000; consensus.nWitnessForkHeight = 700000; + consensus.nAllClaimsInMerkleForkHeight = 10000000; // pick real height consensus.fPowAllowMinDifficultyBlocks = false; consensus.fPowNoRetargeting = false; consensus.nRuleChangeActivationThreshold = 1916; // 95% of a half week @@ -263,6 +264,7 @@ public: consensus.nMinTakeoverWorkaroundHeight = 99; consensus.nMaxTakeoverWorkaroundHeight = 10000000; consensus.nWitnessForkHeight = 1600000; + consensus.nAllClaimsInMerkleForkHeight = 10000000; // pick real height consensus.fPowAllowMinDifficultyBlocks = true; consensus.fPowNoRetargeting = false; consensus.nRuleChangeActivationThreshold = 1512; // 75% for testchains @@ -369,6 +371,7 @@ public: consensus.nMinTakeoverWorkaroundHeight = -1; consensus.nMaxTakeoverWorkaroundHeight = -1; consensus.nWitnessForkHeight = 150; + consensus.nAllClaimsInMerkleForkHeight = 1000; consensus.fPowAllowMinDifficultyBlocks = false; consensus.fPowNoRetargeting = false; consensus.nRuleChangeActivationThreshold = 108; // 75% for testchains diff --git a/src/claimtrie.cpp b/src/claimtrie.cpp index 2e32d2f1e..bed7f8ea1 100644 --- a/src/claimtrie.cpp +++ b/src/claimtrie.cpp @@ -8,7 +8,7 @@ #include #include -static const uint256 one = uint256S("0000000000000000000000000000000000000000000000000000000000000001"); +extern const uint256 one = uint256S("0000000000000000000000000000000000000000000000000000000000000001"); std::vector heightToVch(int n) { @@ -442,9 +442,6 @@ void completeHash(uint256& partialHash, const std::string& key, std::size_t to) .Finalize(partialHash.begin()); } -template -using iCbType = std::function; - bool CClaimTrie::checkConsistency(const uint256& rootHash) const { CClaimTrieDataNode node; @@ -582,7 +579,7 @@ bool CClaimTrieCacheBase::flush() for (const auto& e : claimsToAddToByIdIndex) batch.Write(std::make_pair(CLAIM_BY_ID, e.claim.claimId), e); - auto rootHash = getMerkleHash(); + getMerkleHash(); for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) { bool removed = forDeleteFromBase.erase(it.key()); @@ -621,17 +618,6 @@ bool CClaimTrieCacheBase::flush() } auto ret = base->db->WriteBatch(batch); - // for debugging: -// if (nNextHeight >= 235099) -// { -// g_logger->EnableCategory(BCLog::CLAIMS); -// if (!base->checkConsistency(rootHash)) { -// LogPrintf("Failure with consistency on block height %d\n", nNextHeight); -// dumpToLog(begin()); -// assert(false); -// } -// } - clear(); return ret; } @@ -1337,7 +1323,6 @@ bool CClaimTrieCacheBase::decrementBlock(insertUndoType& insertUndo, claimQueueR undoDecrement(insertSupportUndo, expireSupportUndo); undoDecrement(insertUndo, expireUndo, &claimsToAddToByIdIndex, &claimsToDeleteFromByIdIndex); - return true; } @@ -1446,14 +1431,11 @@ bool CClaimTrieCacheBase::clear() bool CClaimTrieCacheBase::getProofForName(const std::string& name, CClaimTrieProof& proof) { - COutPoint outPoint; // cache the parent nodes cacheData(name, false); getMerkleHash(); - bool fNameHasValue = false; - int nHeightOfLastTakeover = 0; - std::vector nodes; - for (const auto& it : nodesToAddOrUpdate.nodes(name)) { + proof = CClaimTrieProof(); + for (auto& it : static_cast(nodesToAddOrUpdate).nodes(name)) { CClaimValue claim; const auto& key = it.key(); bool fNodeHasValue = it->getBestClaim(claim); @@ -1463,12 +1445,12 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, CClaimTriePro const auto pos = key.size(); std::vector> children; - for (const auto& child : it.children()) { - auto childKey = child.key(); + for (auto& child : it.children()) { + auto& childKey = child.key(); if (name.find(childKey) == 0) { for (auto i = pos; i + 1 < childKey.size(); ++i) { children.emplace_back(childKey[i], uint256{}); - nodes.emplace_back(std::move(children), fNodeHasValue, valueHash); + proof.nodes.emplace_back(children, fNodeHasValue, valueHash); children.clear(); // move promises to leave it in a valid state only valueHash.SetNull(); fNodeHasValue = false; @@ -1481,16 +1463,15 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, CClaimTriePro children.emplace_back(childKey[pos], hash); } if (key == name) { - fNameHasValue = fNodeHasValue; - if (fNameHasValue) { - outPoint = claim.outPoint; - nHeightOfLastTakeover = it->nHeightOfLastTakeover; + proof.hasValue = fNodeHasValue; + if (proof.hasValue) { + proof.outPoint = claim.outPoint; + proof.nHeightOfLastTakeover = it->nHeightOfLastTakeover; } valueHash.SetNull(); } - nodes.emplace_back(std::move(children), fNodeHasValue, valueHash); + proof.nodes.emplace_back(std::move(children), fNodeHasValue, valueHash); } - proof = CClaimTrieProof(std::move(nodes), fNameHasValue, outPoint, nHeightOfLastTakeover); return true; } diff --git a/src/claimtrie.h b/src/claimtrie.h index f39fd32a5..6e45be8ac 100644 --- a/src/claimtrie.h +++ b/src/claimtrie.h @@ -325,12 +325,9 @@ struct CClaimsForNameType class CClaimTrie { - int nNextHeight = 0; - int nProportionalDelayFactor = 0; - std::unique_ptr db; - public: CClaimTrie() = default; + virtual ~CClaimTrie() = default; CClaimTrie(CClaimTrie&&) = delete; CClaimTrie(const CClaimTrie&) = delete; CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor = 32); @@ -347,8 +344,8 @@ public: std::size_t getTotalNamesInTrie() const; std::size_t getTotalClaimsInTrie() const; + virtual bool checkConsistency(const uint256& rootHash) const; CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const; - bool checkConsistency(const uint256& rootHash) const; bool contains(const std::string& key) const; bool empty() const; @@ -357,6 +354,11 @@ public: std::vector> nodes(const std::string& key) const; +protected: + int nNextHeight = 0; + int nProportionalDelayFactor = 0; + std::unique_ptr db; + using recurseNodesCB = void(const std::string&, const CClaimTrieData&, const std::vector&); void recurseNodes(const std::string& name, const CClaimTrieDataNode& current, std::function function) const; }; @@ -381,21 +383,16 @@ struct CClaimTrieProofNode struct CClaimTrieProof { CClaimTrieProof() = default; - - CClaimTrieProof(std::vector nodes, bool hasValue, const COutPoint& outPoint, int nHeightOfLastTakeover) - : nodes(std::move(nodes)), hasValue(hasValue), outPoint(outPoint), nHeightOfLastTakeover(nHeightOfLastTakeover) - { - } - CClaimTrieProof(CClaimTrieProof&&) = default; CClaimTrieProof(const CClaimTrieProof&) = default; CClaimTrieProof& operator=(CClaimTrieProof&&) = default; CClaimTrieProof& operator=(const CClaimTrieProof&) = default; + std::vector> pairs; std::vector nodes; - bool hasValue; + int nHeightOfLastTakeover = 0; + bool hasValue = false; COutPoint outPoint; - int nHeightOfLastTakeover; }; template @@ -518,7 +515,7 @@ public: virtual int expirationTime() const; - bool finalizeDecrement(std::vector>& takeoverHeightUndo); + virtual bool finalizeDecrement(std::vector>& takeoverHeightUndo); virtual CClaimsForNameType getClaimsForName(const std::string& name) const; @@ -536,9 +533,10 @@ public: protected: CClaimTrie* base; CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush + std::unordered_set nodesAlreadyCached; // set of nodes already pulled into cache from base std::unordered_set namesToCheckForTakeover; // takeover numbers are updated on increment - uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it); + virtual uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it); virtual bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover); virtual bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover); @@ -580,7 +578,6 @@ private: std::unordered_map supportCache; // to be added/updated to base (and disk) on flush std::unordered_set nodesToDelete; // to be removed from base (and disk) on flush - std::unordered_set nodesAlreadyCached; // set of nodes already pulled into cache from base std::unordered_map takeoverWorkaround; std::unordered_set removalWorkaround; std::unordered_set forDeleteFromBase; @@ -661,7 +658,8 @@ public: void setExpirationTime(int time); int expirationTime() const override; - void expirationForkActive(int height, bool increment); + virtual void initializeIncrement(); + bool finalizeDecrement(std::vector>& takeoverHeightUndo) override; bool incrementBlock(insertUndoType& insertUndo, claimQueueRowType& expireUndo, @@ -729,6 +727,31 @@ private: std::vector>& takeoverHeightUndo); }; -typedef CClaimTrieCacheNormalizationFork CClaimTrieCache; +class CClaimTrieCacheHashFork : public CClaimTrieCacheNormalizationFork +{ +public: + explicit CClaimTrieCacheHashFork(CClaimTrie* base); + + bool getProofForName(const std::string& name, CClaimTrieProof& proof) override; + bool getProofForName(const std::string& name, CClaimTrieProof& proof, const uint160& claimId); + void initializeIncrement() override; + bool finalizeDecrement(std::vector>& takeoverHeightUndo) override; + +protected: + uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it) override; + +private: + void copyAllBaseToCache(); +}; + +class CClaimTrieHashFork : public CClaimTrie +{ +public: + using CClaimTrie::CClaimTrie; +protected: + bool checkConsistency(const uint256& rootHash) const override; +}; + +typedef CClaimTrieCacheHashFork CClaimTrieCache; #endif // BITCOIN_CLAIMTRIE_H diff --git a/src/claimtrieforks.cpp b/src/claimtrieforks.cpp index 69e89cf1f..4c79a885a 100644 --- a/src/claimtrieforks.cpp +++ b/src/claimtrieforks.cpp @@ -1,6 +1,8 @@ +#include #include #include +#include #include #include @@ -44,10 +46,21 @@ bool CClaimTrieCacheExpirationFork::decrementBlock(insertUndoType& insertUndo, c return false; } -void CClaimTrieCacheExpirationFork::expirationForkActive(int nHeight, bool increment) +void CClaimTrieCacheExpirationFork::initializeIncrement() { - if (nHeight == Params().GetConsensus().nExtendedClaimExpirationForkHeight) - forkForExpirationChange(increment); + // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests) + if (nNextHeight != Params().GetConsensus().nExtendedClaimExpirationForkHeight) + return; + + forkForExpirationChange(true); +} + +bool CClaimTrieCacheExpirationFork::finalizeDecrement(std::vector>& takeoverHeightUndo) +{ + auto ret = CClaimTrieCacheBase::finalizeDecrement(takeoverHeightUndo); + if (ret && nNextHeight == Params().GetConsensus().nExtendedClaimExpirationForkHeight) + forkForExpirationChange(false); + return ret; } bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment) @@ -247,3 +260,245 @@ std::string CClaimTrieCacheNormalizationFork::adjustNameForValidHeight(const std { return normalizeClaimName(name, validHeight > Params().GetConsensus().nNormalizedNameForkHeight); } + +CClaimTrieCacheHashFork::CClaimTrieCacheHashFork(CClaimTrie* base) : CClaimTrieCacheNormalizationFork(base) +{ +} + +static const uint256 leafHash = uint256S("0000000000000000000000000000000000000000000000000000000000000002"); +static const uint256 emptyHash = uint256S("0000000000000000000000000000000000000000000000000000000000000003"); + +std::vector getClaimHashes(const CClaimTrieData& data) +{ + std::vector hashes; + for (auto& claim : data.claims) + hashes.push_back(getValueHash(claim.outPoint, data.nHeightOfLastTakeover)); + return hashes; +} + +template +using iCbType = std::function; + +template +using decay = typename std::decay::type; + +template +uint256 recursiveMerkleHash(const CClaimTrieData& data, Vector&& children, const iCbType& childHash) +{ + static_assert(std::is_same, std::vector>>::value, "Vector should be std vector"); + static_assert(std::is_same::value, "Vector element type should match callback type"); + + std::vector childHashes; + for (auto& child : children) + childHashes.emplace_back(childHash(child)); + + std::vector claimHashes; + if (!data.empty()) + claimHashes = getClaimHashes(data); + else if (children.empty()) + return {}; + + auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes); + auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes); + + return Hash(left.begin(), left.end(), right.begin(), right.end()); +} + +extern const uint256 one; + +bool CClaimTrieHashFork::checkConsistency(const uint256& rootHash) const +{ + if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight) + return CClaimTrie::checkConsistency(rootHash); + + CClaimTrieDataNode node; + if (!find({}, node) || node.hash != rootHash) { + if (rootHash == one) + return true; + + return error("Mismatched root claim trie hashes. This may happen when there is not a clean process shutdown. Please run with -reindex."); + } + + bool success = true; + recurseNodes({}, node, [&success, this](const std::string& name, const CClaimTrieData& data, const std::vector& children) { + if (!success) return; + + iCbType callback = [&success, &name, this](const std::string& child) -> uint256 { + auto key = name + child; + CClaimTrieDataNode node; + success &= find(key, node); + return node.hash; + }; + + success &= !data.hash.IsNull(); + success &= data.hash == recursiveMerkleHash(data, children, callback); + }); + + return success; +} + +uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it) +{ + if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight) + return CClaimTrieCacheNormalizationFork::recursiveComputeMerkleHash(it); + + using iterator = CClaimPrefixTrie::iterator; + iCbType process = [&process](iterator& it) -> uint256 { + if (it->hash.IsNull()) + it->hash = recursiveMerkleHash(it.data(), it.children(), process); + return it->hash; + }; + return process(it); +} + +std::vector ComputeMerklePath(const std::vector& hashes, uint32_t idx) +{ + uint32_t count = 0; + int matchlevel = -1; + bool matchh = false; + uint256 inner[32], h; + const uint32_t one = 1; + std::vector res; + + const auto iterateInner = [&](int& level) { + for (; !(count & (one << level)); level++) { + const auto& ihash = inner[level]; + if (matchh) { + res.push_back(ihash); + } else if (matchlevel == level) { + res.push_back(h); + matchh = true; + } + h = Hash(ihash.begin(), ihash.end(), h.begin(), h.end()); + } + }; + + while (count < hashes.size()) { + h = hashes[count]; + matchh = count == idx; + count++; + int level = 0; + iterateInner(level); + // Store the resulting hash at inner position level. + inner[level] = h; + if (matchh) + matchlevel = level; + } + + int level = 0; + while (!(count & (one << level))) + level++; + + h = inner[level]; + matchh = matchlevel == level; + + while (count != (one << level)) { + // If we reach this point, h is an inner value that is not the top. + if (matchh) + res.push_back(h); + + h = Hash(h.begin(), h.end(), h.begin(), h.end()); + // Increment count to the value it would have if two entries at this + count += (one << level); + level++; + iterateInner(level); + } + return res; +} + +bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof) +{ + return getProofForName(name, proof, uint160()); +} + +bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof, const uint160& claimId) +{ + if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight) + return CClaimTrieCacheNormalizationFork::getProofForName(name, proof); + + auto fillPairs = [&proof](const std::vector& hashes, uint32_t idx) { + auto partials = ComputeMerklePath(hashes, idx); + for (int i = partials.size() - 1; i >= 0; --i) + proof.pairs.emplace_back((idx >> i) & 1, partials[i]); + }; + + // cache the parent nodes + cacheData(name, false); + getMerkleHash(); + proof = CClaimTrieProof(); + for (auto& it : static_cast(nodesToAddOrUpdate).nodes(name)) { + std::vector childHashes; + uint32_t nextCurrentIdx = 0; + for (auto& child : it.children()) { + if (name.find(child.key()) == 0) + nextCurrentIdx = uint32_t(childHashes.size()); + childHashes.push_back(child->hash); + } + + std::vector claimHashes; + if (!it->empty()) + claimHashes = getClaimHashes(it.data()); + + // I am on a node; I need a hash(children, claims) + // if I am the last node on the list, it will be hash(children, x) + // else it will be hash(x, claims) + if (it.key() == name) { + uint32_t nClaimIndex = 0; + auto& claims = it->claims; + auto itClaim = claimId.IsNull() ? claims.begin() : + std::find_if(claims.begin(), claims.end(), [&claimId](const CClaimValue& claim) { + return claim.claimId == claimId; + }); + if (itClaim != claims.end()) { + proof.hasValue = true; + proof.outPoint = itClaim->outPoint; + proof.nHeightOfLastTakeover = it->nHeightOfLastTakeover; + nClaimIndex = std::distance(claims.begin(), itClaim); + } + auto hash = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes); + proof.pairs.emplace_back(true, hash); + if (!claimHashes.empty()) + fillPairs(claimHashes, nClaimIndex); + } else { + auto hash = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes); + proof.pairs.emplace_back(false, hash); + if (!childHashes.empty()) + fillPairs(childHashes, nextCurrentIdx); + } + } + std::reverse(proof.pairs.begin(), proof.pairs.end()); + return true; +} + +void CClaimTrieCacheHashFork::copyAllBaseToCache() +{ + recurseNodes({}, [this](const std::string& name, const CClaimTrieData& data) { + if (nodesAlreadyCached.insert(name).second) + nodesToAddOrUpdate.insert(name, data); + }); + + for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) { + it->hash.SetNull(); + it->flags |= CClaimTrieDataFlags::HASH_DIRTY; + } +} + +void CClaimTrieCacheHashFork::initializeIncrement() +{ + CClaimTrieCacheNormalizationFork::initializeIncrement(); + // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests) + if (nNextHeight != Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1) + return; + + // if we are forking, we load the entire base trie into the cache trie + // we reset its hash computation so it can be recomputed completely + copyAllBaseToCache(); +} + +bool CClaimTrieCacheHashFork::finalizeDecrement(std::vector>& takeoverHeightUndo) +{ + auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement(takeoverHeightUndo); + if (ret && nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1) + copyAllBaseToCache(); + return ret; +} diff --git a/src/consensus/params.h b/src/consensus/params.h index 582159a27..9534d8834 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -96,6 +96,8 @@ struct Params { nOriginalClaimExpirationTime : nExtendedClaimExpirationTime; } + /** blocks before the hard fork that adds all claims into the merkle hash */ + int64_t nAllClaimsInMerkleForkHeight; int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; } uint256 nMinimumChainWork; uint256 defaultAssumeValid; diff --git a/src/init.cpp b/src/init.cpp index 679cbfff8..ae648bf69 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1461,7 +1461,7 @@ bool AppInitMain() pblocktree.reset(); pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); delete pclaimTrie; - pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState); + pclaimTrie = new CClaimTrieHashFork(false, fReindex || fReindexChainState); if (fReset) { pblocktree->WriteReindexing(true); diff --git a/src/miner.cpp b/src/miner.cpp index 9a07b3978..633f4618d 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -71,7 +71,7 @@ void blockToCache(const CBlock* pblock, CClaimTrieCache& trieCache, int nHeight) .claimUndoHeights = {} }; - trieCache.expirationForkActive(nHeight, true); + trieCache.initializeIncrement(); CCoinsViewCache view(pcoinsTip.get()); diff --git a/src/prefixtrie.cpp b/src/prefixtrie.cpp index 3793523f1..09878db8e 100644 --- a/src/prefixtrie.cpp +++ b/src/prefixtrie.cpp @@ -205,6 +205,24 @@ bool CPrefixTrie::find(const TKey& key, TNode node, const callback< return find(TKey(key.begin() + count, key.end()), it->second, cb); } +template +template +std::vector CPrefixTrie::nodes(const TKey& key, TNode root) +{ + std::vector ret; + ret.reserve(1 + key.size()); + ret.emplace_back(TKey{}, root); + if (key.empty()) return ret; + TKey name; + using CBType = callback; + CBType cb = [&name, &ret](const TKey& key, TNode node) { + name.insert(name.end(), key.begin(), key.end()); + ret.emplace_back(name, node); + }; + find(key, root, cb); + return ret; +} + template std::shared_ptr::Node>& CPrefixTrie::insert(const TKey& key, std::shared_ptr::Node>& node) { @@ -368,21 +386,17 @@ TData& CPrefixTrie::at(const TKey& key) } template -std::vector::iterator> CPrefixTrie::nodes(const TKey& key) const +std::vector::iterator> CPrefixTrie::nodes(const TKey& key) { - std::vector ret; - if (empty()) return ret; - ret.reserve(1 + key.size()); - ret.emplace_back(TKey{}, root); - if (key.empty()) return ret; - TKey name; - using CBType = callback>; - CBType cb = [&name, &ret](const TKey& key, std::shared_ptr node) { - name.insert(name.end(), key.begin(), key.end()); - ret.emplace_back(name, node); - }; - find(key, root, cb); - return ret; + if (empty()) return {}; + return nodes(key, root); +} + +template +std::vector::const_iterator> CPrefixTrie::nodes(const TKey& key) const +{ + if (empty()) return {}; + return nodes(key, root); } template @@ -426,7 +440,7 @@ typename CPrefixTrie::iterator CPrefixTrie::begin() template typename CPrefixTrie::iterator CPrefixTrie::end() { - return iterator{TKey(), std::shared_ptr{}}; + return {}; } template @@ -438,7 +452,7 @@ typename CPrefixTrie::const_iterator CPrefixTrie::cbeg template typename CPrefixTrie::const_iterator CPrefixTrie::cend() { - return const_iterator{TKey(), std::shared_ptr{}}; + return {}; } template @@ -450,7 +464,7 @@ typename CPrefixTrie::const_iterator CPrefixTrie::begi template typename CPrefixTrie::const_iterator CPrefixTrie::end() const { - return const_iterator{TKey(), std::shared_ptr{}}; + return {}; } using Key = std::string; diff --git a/src/prefixtrie.h b/src/prefixtrie.h index 142ec4e9d..11d89060f 100644 --- a/src/prefixtrie.h +++ b/src/prefixtrie.h @@ -65,7 +65,7 @@ class CPrefixTrie using difference_type = std::ptrdiff_t; using iterator_category = std::forward_iterator_tag; - Iterator() = delete; + Iterator() = default; Iterator(const Iterator&) = default; Iterator(Iterator&& o) noexcept = default; Iterator(const TKey& name, const std::shared_ptr& node) noexcept; @@ -115,6 +115,9 @@ class CPrefixTrie template static bool find(const TKey& key, TNode node, const callback& cb); + template + static std::vector nodes(const TKey& key, TNode root); + std::shared_ptr& insert(const TKey& key, std::shared_ptr& node); void erase(const TKey& key, std::shared_ptr& node); @@ -140,7 +143,8 @@ public: TData& at(const TKey& key); - std::vector nodes(const TKey& key) const; + std::vector nodes(const TKey& key); + std::vector nodes(const TKey& key) const; bool erase(const TKey& key); diff --git a/src/rpc/claimtrie.cpp b/src/rpc/claimtrie.cpp index 51b229427..defb31bb9 100644 --- a/src/rpc/claimtrie.cpp +++ b/src/rpc/claimtrie.cpp @@ -133,7 +133,7 @@ static bool getValueForOutPoint(const CCoinsViewCache& coinsCache, const COutPoi bool validParams(const UniValue& params, uint8_t required, uint8_t optional) { auto count = params.size(); - return count == required || count == required + optional; + return count >= required && count <= required + optional; } static UniValue getclaimsintrie(const JSONRPCRequest& request) @@ -761,30 +761,43 @@ UniValue proofToJSON(const CClaimTrieProof& proof) { UniValue result(UniValue::VOBJ); UniValue nodes(UniValue::VARR); - for (std::vector::const_iterator itNode = proof.nodes.begin(); itNode != proof.nodes.end(); ++itNode) - { + + for (const auto& itNode : proof.nodes) { UniValue node(UniValue::VOBJ); UniValue children(UniValue::VARR); - for (std::vector >::const_iterator itChildren = itNode->children.begin(); itChildren != itNode->children.end(); ++itChildren) - { + + for (const auto& itChildren : itNode.children) { UniValue child(UniValue::VOBJ); - child.pushKV("character", itChildren->first); - if (!itChildren->second.IsNull()) - { - child.pushKV("nodeHash", itChildren->second.GetHex()); - } + child.pushKV("character", itChildren.first); + if (!itChildren.second.IsNull()) + child.pushKV("nodeHash", itChildren.second.GetHex()); children.push_back(child); } + node.pushKV("children", children); - if (itNode->hasValue && !itNode->valHash.IsNull()) - { - node.pushKV("valueHash", itNode->valHash.GetHex()); - } + + if (itNode.hasValue && !itNode.valHash.IsNull()) + node.pushKV("valueHash", itNode.valHash.GetHex()); + nodes.push_back(node); } - result.pushKV("nodes", nodes); - if (proof.hasValue) - { + + if (!nodes.empty()) + result.push_back(Pair("nodes", nodes)); + + UniValue pairs(UniValue::VARR); + + for (const auto& itPair : proof.pairs) { + UniValue child(UniValue::VOBJ); + child.push_back(Pair("odd", itPair.first)); + child.push_back(Pair("hash", itPair.second.GetHex())); + pairs.push_back(child); + } + + if (!pairs.empty()) + result.push_back(Pair("pairs", pairs)); + + if (proof.hasValue) { result.pushKV("txhash", proof.outPoint.hash.GetHex()); result.pushKV("nOut", (int)proof.outPoint.n); result.pushKV("last takeover height", (int)proof.nHeightOfLastTakeover); @@ -794,7 +807,7 @@ UniValue proofToJSON(const CClaimTrieProof& proof) UniValue getnameproof(const JSONRPCRequest& request) { - if (request.fHelp || !validParams(request.params, 1, 1)) + if (request.fHelp || !validParams(request.params, 1, 2)) throw std::runtime_error( "getnameproof\n" "Return the cryptographic proof that a name maps to a value\n" @@ -807,9 +820,10 @@ UniValue getnameproof(const JSONRPCRequest& request) " none is given, \n" " the latest block\n" " will be used.\n" + "3. \"claimId\" (string, optional, post-fork) for validating a specific claim\n" "Result: \n" "{\n" - " \"nodes\" : [ (array of object) full nodes (i.e.\n" + " \"nodes\" : [ (array of object, pre-fork) full nodes (i.e.\n" " those which lead to\n" " the requested name)\n" " \"children\" : [ (array of object) the children of\n" @@ -835,6 +849,13 @@ UniValue getnameproof(const JSONRPCRequest& request) " the node has a\n" " value or not\n" " ]\n" + " \"pairs\" : [ (array of pairs, post-fork) hash can be validated by \n" + " hashing claim from the bottom up\n" + " {\n" + " \"odd\" (boolean) this value goes on the right of hash\n" + " \"hash\" (boolean) the hash to be mixed in\n" + " }\n" + " ]\n" " \"txhash\" : \"hash\" (string, if exists) the txid of the\n" " claim which controls\n" " this name, if there\n" @@ -855,14 +876,18 @@ UniValue getnameproof(const JSONRPCRequest& request) CCoinsViewCache coinsCache(pcoinsTip.get()); CClaimTrieCache trieCache(pclaimTrie); - if (request.params.size() == 2) { + if (request.params.size() > 1) { CBlockIndex* pblockIndex = BlockHashIndex(ParseHashV(request.params[1], "blockhash (optional parameter 2)")); RollBackTo(pblockIndex, coinsCache, trieCache); } + uint160 claimId; + if (request.params.size() > 2) + claimId = ParseClaimtrieId(request.params[2], "claimId (optional parameter 3)"); + CClaimTrieProof proof; std::string name = request.params[0].get_str(); - if (!trieCache.getProofForName(name, proof)) + if (!trieCache.getProofForName(name, proof, claimId)) throw JSONRPCError(RPC_INTERNAL_ERROR, "Failed to generate proof"); return proofToJSON(proof); @@ -898,7 +923,7 @@ static const CRPCCommand commands[] = { "Claimtrie", "gettotalclaims", &gettotalclaims, { "" } }, { "Claimtrie", "gettotalvalueofclaims", &gettotalvalueofclaims, { "controlling_only" } }, { "Claimtrie", "getclaimsfortx", &getclaimsfortx, { "txid" } }, - { "Claimtrie", "getnameproof", &getnameproof, { "name","blockhash"} }, + { "Claimtrie", "getnameproof", &getnameproof, { "name","blockhash","claimId"} }, { "Claimtrie", "getclaimbyid", &getclaimbyid, { "claimId" } }, { "Claimtrie", "checknormalization", &checknormalization, { "name" }}, }; diff --git a/src/test/claimtriebranching_tests.cpp b/src/test/claimtriebranching_tests.cpp index ae74ea6c5..b14daa871 100644 --- a/src/test/claimtriebranching_tests.cpp +++ b/src/test/claimtriebranching_tests.cpp @@ -72,7 +72,7 @@ static BlockAssembler AssemblerForTest() } // Test Fixtures -struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork +struct ClaimTrieChainFixture: public CClaimTrieCache { std::vector coinbase_txs; std::vector marks; @@ -85,11 +85,12 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork int64_t expirationForkHeight; int64_t originalExpiration; int64_t extendedExpiration; + int64_t forkhash_original; - using CClaimTrieCacheExpirationFork::getSupportsForName; + using CClaimTrieCache::getSupportsForName; - ClaimTrieChainFixture(): CClaimTrieCacheExpirationFork(pclaimTrie), - unique_block_counter(0), normalization_original(-1), expirationForkHeight(-1) + ClaimTrieChainFixture(): CClaimTrieCache(pclaimTrie), + unique_block_counter(0), normalization_original(-1), expirationForkHeight(-1), forkhash_original(-1) { fRequireStandard = false; BOOST_CHECK_EQUAL(nNextHeight, chainActive.Height() + 1); @@ -121,6 +122,9 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork consensus.nExtendedClaimExpirationTime = extendedExpiration; consensus.nOriginalClaimExpirationTime = originalExpiration; } + if (forkhash_original >= 0) { + consensus.nAllClaimsInMerkleForkHeight = forkhash_original; + } } void setExpirationForkHeight(int targetMinusCurrent, int64_t preForkExpirationTime, int64_t postForkExpirationTime) { @@ -145,6 +149,15 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork consensus.nNormalizedNameForkHeight = target; } + void setHashForkHeight(int targetMinusCurrent) + { + int target = chainActive.Height() + targetMinusCurrent; + auto& consensus = const_cast(Params().GetConsensus()); + if (forkhash_original < 0) + forkhash_original = consensus.nAllClaimsInMerkleForkHeight; + consensus.nAllClaimsInMerkleForkHeight = target; + } + bool CreateBlock(const std::unique_ptr& pblocktemplate) { CBlock* pblock = &pblocktemplate->block; @@ -1422,8 +1435,8 @@ BOOST_AUTO_TEST_CASE(claimtriebranching_normalization) BOOST_CHECK(fixture.is_best_claim("normalizetest", tx1)); BOOST_CHECK(fixture.best_claim_effective_amount_equals("normalizetest", 3)); - CClaimValue val; - BOOST_CHECK(!fixture.getInfoForName("normalizeTest", val)); + CClaimTrieData data; + BOOST_CHECK(!pclaimTrie->find("normalizeTest", data)); // Check equivalence of normalized claim names BOOST_CHECK(fixture.is_best_claim("normalizetest", tx1)); // collapsed tx2 @@ -1552,7 +1565,8 @@ BOOST_AUTO_TEST_CASE(claimtriecache_normalization) BOOST_CHECK(!trieCache.spendClaim(name_normd, COutPoint(tx2.GetHash(), 0), currentHeight, amelieValidHeight)); BOOST_CHECK(trieCache.spendClaim(name_upper, COutPoint(tx2.GetHash(), 0), currentHeight, amelieValidHeight)); - BOOST_CHECK(!fixture.getInfoForName(name, nval1)); + CClaimTrieData data; + BOOST_CHECK(!pclaimTrie->find(name, data)); BOOST_CHECK(trieCache.getInfoForName(name, nval1)); BOOST_CHECK(trieCache.addClaim(name, COutPoint(tx1.GetHash(), 0), ClaimIdHash(tx1.GetHash(), 0), CAmount(2), currentHeight + 1)); BOOST_CHECK(trieCache.getInfoForName(name, nval1)); @@ -3597,10 +3611,10 @@ bool verify_proof(const CClaimTrieProof proof, uint256 rootHash, const std::stri std::string computedReverseName; bool verifiedValue = false; - for (std::vector::const_reverse_iterator itNodes = proof.nodes.rbegin(); itNodes != proof.nodes.rend(); ++itNodes) { + for (auto itNodes = proof.nodes.rbegin(); itNodes != proof.nodes.rend(); ++itNodes) { bool foundChildInChain = false; std::vector vchToHash; - for (std::vector >::const_iterator itChildren = itNodes->children.begin(); itChildren != itNodes->children.end(); ++itChildren) { + for (auto itChildren = itNodes->children.begin(); itChildren != itNodes->children.end(); ++itChildren) { vchToHash.push_back(itChildren->first); uint256 childHash; if (itChildren->second.IsNull()) { @@ -4088,7 +4102,6 @@ BOOST_AUTO_TEST_CASE(claim_rpcs_rollback3_test) UniValue valueResults = getvalueforname(req); BOOST_CHECK_EQUAL(valueResults["value"].get_str(), HexStr(sValue1)); BOOST_CHECK_EQUAL(valueResults["amount"].get_int(), 3); - } BOOST_AUTO_TEST_CASE(update_on_support2_test) @@ -4118,4 +4131,141 @@ BOOST_AUTO_TEST_CASE(update_on_support2_test) BOOST_CHECK_EQUAL(node.nHeightOfLastTakeover, height + 1); } +void ValidatePairs(CClaimTrieCache& cache, const CClaimTrieProof& proof, uint256 claimHash) +{ + for (auto& pair : proof.pairs) + if (pair.first) // we're on the right because we were an odd index number + claimHash = Hash(pair.second.begin(), pair.second.end(), claimHash.begin(), claimHash.end()); + else + claimHash = Hash(claimHash.begin(), claimHash.end(), pair.second.begin(), pair.second.end()); + + BOOST_CHECK_EQUAL(cache.getMerkleHash(), claimHash); +} + +BOOST_AUTO_TEST_CASE(hash_includes_all_claims_rollback_test) +{ + ClaimTrieChainFixture fixture; + fixture.setHashForkHeight(5); + + CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 1); + fixture.IncrementBlocks(1); + + uint256 currentRoot = fixture.getMerkleHash(); + fixture.IncrementBlocks(1); + BOOST_CHECK_EQUAL(currentRoot, fixture.getMerkleHash()); + fixture.IncrementBlocks(3); + BOOST_CHECK_NE(currentRoot, fixture.getMerkleHash()); + fixture.DecrementBlocks(3); + BOOST_CHECK_EQUAL(currentRoot, fixture.getMerkleHash()); +} + +BOOST_AUTO_TEST_CASE(hash_includes_all_claims_single_test) +{ + ClaimTrieChainFixture fixture; + fixture.setHashForkHeight(2); + fixture.IncrementBlocks(4); + + CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 1); + fixture.IncrementBlocks(1); + + COutPoint outPoint(tx1.GetHash(), 0); + uint160 claimId = ClaimIdHash(tx1.GetHash(), 0); + + CClaimTrieProof proof; + BOOST_CHECK(fixture.getProofForName("test", proof, claimId)); + BOOST_CHECK(proof.hasValue); + BOOST_CHECK_EQUAL(proof.outPoint, outPoint); + auto claimHash = getValueHash(outPoint, proof.nHeightOfLastTakeover); + ValidatePairs(fixture, proof, claimHash); +} + +BOOST_AUTO_TEST_CASE(hash_includes_all_claims_triple_test) +{ + ClaimTrieChainFixture fixture; + fixture.setHashForkHeight(2); + fixture.IncrementBlocks(4); + + std::string names[] = {"test", "tester", "tester2"}; + CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "one", 1); + CMutableTransaction tx2 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "two", 2); + CMutableTransaction tx3 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "thr", 3); + CMutableTransaction tx7 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "for", 4); + CMutableTransaction tx8 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "fiv", 5); + CMutableTransaction tx4 = fixture.MakeClaim(fixture.GetCoinbase(), names[1], "two", 2); + CMutableTransaction tx5 = fixture.MakeClaim(fixture.GetCoinbase(), names[1], "thr", 3); + CMutableTransaction tx6 = fixture.MakeClaim(fixture.GetCoinbase(), names[2], "one", 1); + fixture.IncrementBlocks(1); + + for (const auto& name : names) { + for (auto& claim : fixture.getClaimsForName(name).claims) { + CClaimTrieProof proof; + BOOST_CHECK(fixture.getProofForName(name, proof, claim.claimId)); + BOOST_CHECK(proof.hasValue); + BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint); + uint256 claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover); + ValidatePairs(fixture, proof, claimHash); + } + } +} + +BOOST_AUTO_TEST_CASE(hash_includes_all_claims_branched_test) +{ + ClaimTrieChainFixture fixture; + fixture.setHashForkHeight(2); + fixture.IncrementBlocks(4); + + std::string names[] = {"test", "toast", "tot", "top", "toa", "toad"}; + for (const auto& name : names) + fixture.MakeClaim(fixture.GetCoinbase(), name, "one", 1); + + fixture.MakeClaim(fixture.GetCoinbase(), "toa", "two", 2); + fixture.MakeClaim(fixture.GetCoinbase(), "toa", "tre", 3); + fixture.MakeClaim(fixture.GetCoinbase(), "toa", "qua", 4); + fixture.MakeClaim(fixture.GetCoinbase(), "toa", "cin", 5); + fixture.IncrementBlocks(1); + + for (const auto& name : names) { + for (auto& claim : fixture.getClaimsForName(name).claims) { + CClaimTrieProof proof; + BOOST_CHECK(fixture.getProofForName(name, proof, claim.claimId)); + BOOST_CHECK(proof.hasValue); + BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint); + uint256 claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover); + ValidatePairs(fixture, proof, claimHash); + } + } +} + +BOOST_AUTO_TEST_CASE(hash_claims_children_fuzzer_test) +{ + ClaimTrieChainFixture fixture; + fixture.setHashForkHeight(2); + fixture.IncrementBlocks(4); + + std::size_t i = 0; + auto names = random_strings(300); + auto lastTx = MakeTransactionRef(fixture.GetCoinbase()); + for (const auto& name : names) { + auto tx = fixture.MakeClaim(*lastTx, name, "one", 1); + lastTx = MakeTransactionRef(std::move(tx)); + if (++i % 5 == 0) + for (std::size_t j = 0; j < (i / 5); ++j) { + auto tx = fixture.MakeClaim(*lastTx, name, "one", 1); + lastTx = MakeTransactionRef(std::move(tx)); + } + fixture.IncrementBlocks(1); + } + + for (const auto& name : names) { + for (auto& claim : fixture.getClaimsForName(name).claims) { + CClaimTrieProof proof; + BOOST_CHECK(fixture.getProofForName(name, proof, claim.claimId)); + BOOST_CHECK(proof.hasValue); + BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint); + uint256 claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover); + ValidatePairs(fixture, proof, claimHash); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/claimtriecache_tests.cpp b/src/test/claimtriecache_tests.cpp index 895e4fd34..38906a9b8 100644 --- a/src/test/claimtriecache_tests.cpp +++ b/src/test/claimtriecache_tests.cpp @@ -297,7 +297,6 @@ BOOST_AUTO_TEST_CASE(iteratetrie_test) ctc.insertClaimIntoTrie("test", claimVal, true); BOOST_CHECK(ctc.flush()); - std::size_t count = 0; CClaimTrieDataNode node; BOOST_CHECK(pclaimTrie->find("", node)); BOOST_CHECK_EQUAL(node.children.size(), 1U); diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 9cae2551a..d7d52d680 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -144,7 +144,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha pblocktree.reset(new CBlockTreeDB(1 << 20, true)); pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - pclaimTrie = new CClaimTrie(true, false, 1); + pclaimTrie = new CClaimTrieHashFork(true, false, 1); if (!LoadGenesisBlock(chainparams)) { throw std::runtime_error("LoadGenesisBlock failed."); } diff --git a/src/validation.cpp b/src/validation.cpp index f141fc67c..782c863c8 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1572,8 +1572,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI assert(merkleHash == pindex->pprev->hashClaimTrie); } - trieCache.expirationForkActive(pindex->nHeight, false); - return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } @@ -1931,8 +1929,6 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // Get the script flags for this block unsigned int flags = GetBlockScriptFlags(pindex, chainparams.GetConsensus()); - trieCache.expirationForkActive(pindex->nHeight, true); - int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime2 - nTime1), nTimeForks * MICRO, nTimeForks * MILLI / nBlocksTotal); @@ -1940,6 +1936,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl CCheckQueueControl control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : nullptr); + trieCache.initializeIncrement(); + std::vector prevheights; CAmount nFees = 0; int nInputs = 0;