From 217c102f8f9d77720d665c02ee903f52a55075d8 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Tue, 12 Nov 2019 18:54:16 -0500 Subject: [PATCH] runs to norm fork --- src/claimscriptop.cpp | 6 +-- src/claimtrie.cpp | 71 ++++++++++++--------------- src/claimtrie.h | 2 +- src/test/claimtriehashfork_tests.cpp | 72 ++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 45 deletions(-) diff --git a/src/claimscriptop.cpp b/src/claimscriptop.cpp index 6c1f84401..ff99bd711 100644 --- a/src/claimscriptop.cpp +++ b/src/claimscriptop.cpp @@ -96,7 +96,7 @@ bool CClaimScriptSpendOp::claimName(CClaimTrieCache& trieCache, const std::strin { auto claimId = ClaimIdHash(point.hash, point.n); auto ret = spendClaim(trieCache, name, claimId); - LogPrint(BCLog::CLAIMS, "--- Spending original claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", + LogPrint(BCLog::CLAIMS, "--- Spent original claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight); return ret; } @@ -105,7 +105,7 @@ bool CClaimScriptSpendOp::updateClaim(CClaimTrieCache& trieCache, const std::str const std::vector& metadata) { auto ret = spendClaim(trieCache, name, claimId); - LogPrint(BCLog::CLAIMS, "--- Spending updated claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", + LogPrint(BCLog::CLAIMS, "--- Spent updated claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight); return ret; } @@ -124,7 +124,7 @@ bool CClaimScriptSpendOp::supportClaim(CClaimTrieCache& trieCache, const std::st { std::string nodeName; bool res = trieCache.removeSupport(point, nodeName, nValidHeight); - LogPrint(BCLog::CLAIMS, "--- Spending support: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", + LogPrint(BCLog::CLAIMS, "--- Spent support: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight); if (!res) LogPrint(BCLog::CLAIMS, "Remove support failed for %s with claimid %s\n", name, claimId.GetHex().substr(0, 6)); diff --git a/src/claimtrie.cpp b/src/claimtrie.cpp index 0639e0a2d..e950769ce 100644 --- a/src/claimtrie.cpp +++ b/src/claimtrie.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -80,7 +81,7 @@ CClaimTrie::CClaimTrie(bool fWipe, int height, int proportionalDelayFactor) db << "CREATE INDEX IF NOT EXISTS supports_nodeName ON supports (nodeName)"; db << "PRAGMA cache_size=-" + std::to_string(5 * 1024); // in -KB - db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit + db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit db << "PRAGMA journal_mode=MEMORY"; db << "PRAGMA temp_store=MEMORY"; db << "PRAGMA case_sensitive_like=true"; @@ -171,15 +172,12 @@ bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COut return false; } -bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::string& parent, std::vector& claims) { +bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims) { if (name.empty()) return false; // to remove a node it must have one or less children and no claims - claims.clear(); - db << "SELECT name FROM claims WHERE nodeName = ?1 AND validHeight < ?2 AND expirationHeight >= ?2 " - << name << nNextHeight >> [&claims](std::string name) { - claims.push_back(std::move(name)); - }; - if (!claims.empty()) return false; // still has claims + db << "SELECT COUNT(*) FROM claims WHERE nodeName = ?1 AND validHeight < ?2 AND expirationHeight >= ?2 " + << name << nNextHeight >> claims; + if (claims > 0) return false; // still has claims // we now know it has no claims, but we need to check its children int64_t count; std::string childName; @@ -204,19 +202,6 @@ bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::str return ret; } -#include - -std::string bin2hex(const std::string& bin) -{ - std::stringstream s; - s << std::hex; - for (uint8_t i : bin) { - if (i < 16) s << '0'; - s << uint32_t(i); - } - return s.str(); -} - void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { if (!transacting) return; @@ -251,11 +236,11 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { for (auto& name: names) { - std::vector claims; + int64_t claims; std::string parent, node; for (node = name; deleteNodeIfPossible(node, parent, claims);) node = parent; - if (node != name || name.empty() || claims.empty()) + if (node != name || name.empty() || claims <= 0) continue; // if you have no claims but we couldn't delete you, you must have legitimate children parentQuery << name.substr(0, name.size() - 1); @@ -275,6 +260,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { }; std::size_t splitPos = 0; auto psize = parent.size() + 1; + const static std::string charsThatBreakLikeOp("_%\0", 3); for (auto& sibling: siblings) { if (sibling.compare(0, psize, name, 0, psize) == 0) { splitPos = psize; @@ -294,7 +280,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { insertQuery << newNodeName << parent; insertQuery++; - if (newNodeName.find('_') == std::string::npos && newNodeName.find('%') == std::string::npos) { + if (newNodeName.find_first_of(charsThatBreakLikeOp) == std::string::npos) { updateUnaffectedsQuery << newNodeName << newNodeName + "_%" << newNodeName.size(); updateUnaffectedsQuery++; } @@ -308,22 +294,19 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { } } - LogPrint(BCLog::CLAIMS, "Inserting or updating node %s(%s), parent %s\n", name, bin2hex(name), parent); + LogPrint(BCLog::CLAIMS, "Inserting or updating node %s (%s), parent %s\n", name, HexStr(name), parent); insertQuery << name << parent; insertQuery++; if (splitPos == 0) db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent; - if (name.find('_') == std::string::npos && name.find('%') == std::string::npos) { + if (name.find_first_of(charsThatBreakLikeOp) == std::string::npos) { updateUnaffectedsQuery << name << name + "_%" << name.size(); updateUnaffectedsQuery++; } else db << "UPDATE nodes SET parent = ?1 WHERE SUBSTR(name, 1, ?2) = ?1 AND LENGTH(parent) < ?2 AND name != ?1" << name << name.size(); - - if (name.size() == 1 && name[0] == 0) - db << "UPDATE nodes SET parent = '' WHERE name = ?" << name; } parentQuery.used(true); @@ -334,7 +317,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { // parents should all be set right db << "UPDATE nodes SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS " "(SELECT parent FROM nodes WHERE hash IS NULL UNION SELECT parent FROM prefix, nodes " - "WHERE name = prefix.p AND prefix.p IS NOT NULL ORDER BY parent DESC) SELECT p FROM prefix)"; + "WHERE name = prefix.p AND prefix.p != '') SELECT p FROM prefix)"; } std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const @@ -357,7 +340,7 @@ std::size_t CClaimTrieCacheBase::getTotalClaimsInTrie() const CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) const { CAmount ret = 0; - std::string query("SELECT IFNULL(SUM(s.amount),0)+c.amount FROM supports s " + std::string query("SELECT SUM(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM supports s " "WHERE s.supportedClaimID = c.claimID AND s.validHeight < ?1 AND s.expirationHeight >= ?1) " "FROM claims c WHERE c.validHeight < ?1 AND s.expirationHeight >= ?1"); if (fControllingOnly) @@ -454,7 +437,7 @@ uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(const std::string& name, if (child.hash->IsNull()) { *child.hash = recursiveComputeMerkleHash(child.name, child.takeoverHeight, checkOnly); } - LogPrint(BCLog::CLAIMS, "Hash of %s(%s): %s, takeover: %d\n", child.name, bin2hex(child.name), (*child.hash).GetHex(), child.takeoverHeight); + LogPrint(BCLog::CLAIMS, "Using hash of %s (%s): %s, takeover: %d\n", child.name, HexStr(child.name), (*child.hash).GetHex(), child.takeoverHeight); completeHash(*child.hash, child.name, pos); vchToHash.push_back(child.name[pos]); vchToHash.insert(vchToHash.end(), child.hash->begin(), child.hash->end()); @@ -547,7 +530,7 @@ CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) nNextHeight = base->nNextHeight; db << "PRAGMA cache_size=-" + std::to_string(200 * 1024); // in -KB - db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit + db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit db << "PRAGMA journal_mode=MEMORY"; db << "PRAGMA temp_store=MEMORY"; db << "PRAGMA case_sensitive_like=true"; @@ -606,7 +589,10 @@ bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& out db << "INSERT INTO claims(claimID, name, nodeName, txID, txN, amount, blockHeight, validHeight, expirationHeight, metadata) " "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << claimId << name << nodeName << outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << expires << metadata; - db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; + + if (nValidHeight < nNextHeight) + db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; + return true; } @@ -623,7 +609,9 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << supportedClaimId << name << nodeName << outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << expires << metadata; - db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; + if (nValidHeight < nNextHeight) + db << "UPDATE nodes SET hash = NULL WHERE name = ?" << nodeName; + return true; } @@ -961,10 +949,11 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimUndoTy // for all dirty nodes look for new takeovers if (!transacting) { transacting = true; db << "begin"; } + db << "INSERT INTO nodes(name) SELECT nodeName FROM claims " + "WHERE validHeight = ?1 AND expirationHeight > ?1 " + "ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight; db << "UPDATE nodes SET hash = NULL WHERE name IN " - "(SELECT nodeName FROM claims WHERE validHeight = ?)" << nNextHeight; - db << "UPDATE nodes SET hash = NULL WHERE name IN " - "(SELECT nodeName FROM supports WHERE validHeight = ?)" << nNextHeight; + "(SELECT nodeName FROM supports WHERE validHeight = ?1 AND expirationHeight > ?1)" << nNextHeight; assert(expireUndo.empty()); { @@ -1027,7 +1016,7 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimUndoTy if (takeoverHappening && activateAllFor(insertUndo, insertSupportUndo, nameWithTakeover)) hasCandidate = getInfoForName(nameWithTakeover, candidateValue, 1); - LogPrint(BCLog::CLAIMS, "Takeover on %s(%s) at %d, happening: %d, set before: %d\n", nameWithTakeover, bin2hex(nameWithTakeover), nNextHeight, takeoverHappening, hasBeenSetBefore); + LogPrint(BCLog::CLAIMS, "Takeover on %s (%s) at %d, happening: %d, set before: %d\n", nameWithTakeover, HexStr(nameWithTakeover), nNextHeight, takeoverHappening, hasBeenSetBefore); if (takeoverHappening || !hasBeenSetBefore) { takeoverUndo.emplace_back(nameWithTakeover, std::make_pair(existingHeight, hasBeenSetBefore ? *existingID : uint160())); @@ -1129,9 +1118,9 @@ bool CClaimTrieCacheBase::decrementBlock(insertUndoType& insertUndo, claimUndoTy bool CClaimTrieCacheBase::finalizeDecrement(takeoverUndoType& takeoverUndo) { db << "UPDATE nodes SET hash = NULL WHERE name IN " - "(SELECT nodeName FROM claims WHERE validHeight = ?)" << nNextHeight; + "(SELECT nodeName FROM claims WHERE validHeight = ?1 AND expirationHeight > ?1)" << nNextHeight; db << "UPDATE nodes SET hash = NULL WHERE name IN " - "(SELECT nodeName FROM supports WHERE validHeight = ?)" << nNextHeight; + "(SELECT nodeName FROM supports WHERE validHeight = ?1 AND expirationHeight > ?1)" << nNextHeight; for (auto it = takeoverUndo.crbegin(); it != takeoverUndo.crend(); ++it) { if (it->second.second.IsNull()) diff --git a/src/claimtrie.h b/src/claimtrie.h index 47cb0c6a3..1e93ef03f 100644 --- a/src/claimtrie.h +++ b/src/claimtrie.h @@ -392,7 +392,7 @@ protected: virtual int getDelayForName(const std::string& name, const uint160& claimId) const; - bool deleteNodeIfPossible(const std::string& name, std::string& parent, std::vector& claims); + bool deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims); void ensureTreeStructureIsUpToDate(); private: diff --git a/src/test/claimtriehashfork_tests.cpp b/src/test/claimtriehashfork_tests.cpp index 3ce8749d2..b19344825 100644 --- a/src/test/claimtriehashfork_tests.cpp +++ b/src/test/claimtriehashfork_tests.cpp @@ -401,4 +401,76 @@ BOOST_AUTO_TEST_CASE(bogus_claimtrie_hash_test) BOOST_CHECK_EQUAL(orig_chain_height, chainActive.Height()); } +#ifndef MAC_OSX // can't find a random number generator that produces the same sequence on OSX + BOOST_AUTO_TEST_CASE(triehash_fuzzer_test) + { + ClaimTrieChainFixture fixture; + + auto envClaims = std::getenv("TRIEHASH_FUZZER_CLAIMS"); + auto envBlocks = std::getenv("TRIEHASH_FUZZER_BLOCKS"); + + const int claimsPerBlock = envClaims ? std::atoi(envClaims) : 100; + const int blocks = envBlocks ? std::atoi(envBlocks) : 13; + + auto names = random_strings(blocks * claimsPerBlock); + + FastRandomContext frc(true); + std::unordered_map> existingClaims; + std::vector existingSupports; + std::string value(1024, 'c'); + + std::vector cb {fixture.GetCoinbase()}; + for (int i = 0; i < blocks; ++i) { + for (int j = 0; j < claimsPerBlock; ++j) { + auto name = names[i * claimsPerBlock + j]; + auto supportFront = frc.randrange(4) == 0; + auto supportBack = frc.randrange(4) == 0; + auto removeClaim = frc.randrange(4) == 0; + auto removeSupport = frc.randrange(4) == 0; + auto hit = existingClaims.find(name); + if (supportFront && hit != existingClaims.end() && hit->second.size()) { + auto tx = fixture.MakeSupport(cb.back(), hit->second[frc.rand64() % hit->second.size()], name, 2); + existingSupports.push_back(tx); + cb.emplace_back(std::move(tx)); + } + if (removeClaim && hit != existingClaims.end() && hit->second.size()) { + auto idx = frc.rand64() % hit->second.size(); + fixture.Spend(hit->second[idx]); + hit->second.erase(hit->second.begin() + idx); + } else { + auto tx = fixture.MakeClaim(cb.back(), name, value, 2); + existingClaims[name].push_back(tx); + hit = existingClaims.find(name); + cb.emplace_back(std::move(tx)); + } + if (supportBack && hit != existingClaims.end() && hit->second.size()) { + auto tx = fixture.MakeSupport(cb.back(), hit->second[frc.rand64() % hit->second.size()], name, 2); + existingSupports.push_back(tx); + cb.emplace_back(std::move(tx)); + } + if (removeSupport && (i & 7) == 7 && !existingSupports.empty()) { + const auto tidx = frc.rand64() % existingSupports.size(); + const auto tx = existingSupports[tidx]; + fixture.Spend(tx); + existingSupports.erase(existingSupports.begin() + tidx); + } + if (cb.back().GetValueOut() < 10 || cb.size() > 40000) { + cb.clear(); + cb.push_back(fixture.GetCoinbase()); + } + } + fixture.IncrementBlocks(1); + if (blocks > 13 && i % 50 == 0) // travisCI needs some periodic output + std::cerr << "In triehash_fuzzer_test with " << fixture.getTotalNamesInTrie() << " names at block " << i << std::endl; + } + + if (blocks == 1000 && claimsPerBlock == 100) + BOOST_CHECK_EQUAL(fixture.getMerkleHash().GetHex(), "28825257a129eef69cab87d6255c8359fc6dc083ca7f09222526e3a7971f382d"); + else if (blocks == 13 && claimsPerBlock == 100) + BOOST_CHECK_EQUAL(fixture.getMerkleHash().GetHex(), "4e5984d6984f5f05d50e821e6228d56bcfbd16ca2093cd0308f6ff1c2bc8689a"); + else + std::cerr << "Hash: " << fixture.getMerkleHash().GetHex() << std::endl; + } +#endif + BOOST_AUTO_TEST_SUITE_END()