LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

runs to norm fork

Browse source
This commit is contained in:
Brannon King 2019-11-12 18:54:16 -05:00 committed by Anthony Fieroni
parent ef244ca801
commit 217c102f8f
4 changed files with 106 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/claimscriptop.cpp
View file

@ -96,7 +96,7 @@ bool CClaimScriptSpendOp::claimName(CClaimTrieCache& trieCache, const std::strin
{ {
auto claimId = ClaimIdHash(point.hash, point.n); auto claimId = ClaimIdHash(point.hash, point.n);
auto ret = spendClaim(trieCache, name, claimId); auto ret = spendClaim(trieCache, name, claimId);
LogPrint(BCLog::CLAIMS, "--- Spending original claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", LogPrint(BCLog::CLAIMS, "--- Spent original claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight); name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
return ret; return ret;
} }
@ -105,7 +105,7 @@ bool CClaimScriptSpendOp::updateClaim(CClaimTrieCache& trieCache, const std::str
const std::vector<unsigned char>& metadata) const std::vector<unsigned char>& metadata)
{ {
auto ret = spendClaim(trieCache, name, claimId); auto ret = spendClaim(trieCache, name, claimId);
LogPrint(BCLog::CLAIMS, "--- Spending updated claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", LogPrint(BCLog::CLAIMS, "--- Spent updated claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight); name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
return ret; return ret;
} }
@ -124,7 +124,7 @@ bool CClaimScriptSpendOp::supportClaim(CClaimTrieCache& trieCache, const std::st
{ {
std::string nodeName; std::string nodeName;
bool res = trieCache.removeSupport(point, nodeName, nValidHeight); bool res = trieCache.removeSupport(point, nodeName, nValidHeight);
LogPrint(BCLog::CLAIMS, "--- Spending support: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n", LogPrint(BCLog::CLAIMS, "--- Spent support: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight); name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
if (!res) if (!res)
LogPrint(BCLog::CLAIMS, "Remove support failed for %s with claimid %s\n", name, claimId.GetHex().substr(0, 6)); LogPrint(BCLog::CLAIMS, "Remove support failed for %s with claimid %s\n", name, claimId.GetHex().substr(0, 6));

69
src/claimtrie.cpp
View file

@ -2,6 +2,7 @@
#include <hash.h> #include <hash.h>
#include <logging.h> #include <logging.h>
#include <util.h> #include <util.h>
#include <utilstrencodings.h>
#include <algorithm> #include <algorithm>
#include <memory> #include <memory>
@ -80,7 +81,7 @@ CClaimTrie::CClaimTrie(bool fWipe, int height, int proportionalDelayFactor)
db << "CREATE INDEX IF NOT EXISTS supports_nodeName ON supports (nodeName)"; db << "CREATE INDEX IF NOT EXISTS supports_nodeName ON supports (nodeName)";
db << "PRAGMA cache_size=-" + std::to_string(5 * 1024); // in -KB db << "PRAGMA cache_size=-" + std::to_string(5 * 1024); // in -KB
db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit
db << "PRAGMA journal_mode=MEMORY"; db << "PRAGMA journal_mode=MEMORY";
db << "PRAGMA temp_store=MEMORY"; db << "PRAGMA temp_store=MEMORY";
db << "PRAGMA case_sensitive_like=true"; db << "PRAGMA case_sensitive_like=true";
@ -171,15 +172,12 @@ bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COut
return false; return false;
} }
bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::string& parent, std::vector<std::string>& claims) { bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims) {
if (name.empty()) return false; if (name.empty()) return false;
// to remove a node it must have one or less children and no claims // to remove a node it must have one or less children and no claims
claims.clear(); db << "SELECT COUNT(*) FROM claims WHERE nodeName = ?1 AND validHeight < ?2 AND expirationHeight >= ?2 "
db << "SELECT name FROM claims WHERE nodeName = ?1 AND validHeight < ?2 AND expirationHeight >= ?2 " << name << nNextHeight >> claims;
<< name << nNextHeight >> [&claims](std::string name) { if (claims > 0) return false; // still has claims
claims.push_back(std::move(name));
};
if (!claims.empty()) return false; // still has claims
// we now know it has no claims, but we need to check its children // we now know it has no claims, but we need to check its children
int64_t count; int64_t count;
std::string childName; std::string childName;
@ -204,19 +202,6 @@ bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::str
return ret; return ret;
} }
#include <sstream>
std::string bin2hex(const std::string& bin)
{
std::stringstream s;
s << std::hex;
for (uint8_t i : bin) {
if (i < 16) s << '0';
s << uint32_t(i);
}
return s.str();
}
void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
if (!transacting) return; if (!transacting) return;
@ -251,11 +236,11 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
for (auto& name: names) { for (auto& name: names) {
std::vector<std::string> claims; int64_t claims;
std::string parent, node; std::string parent, node;
for (node = name; deleteNodeIfPossible(node, parent, claims);) for (node = name; deleteNodeIfPossible(node, parent, claims);)
node = parent; node = parent;
if (node != name || name.empty() || claims.empty()) if (node != name || name.empty() || claims <= 0)
continue; // if you have no claims but we couldn't delete you, you must have legitimate children continue; // if you have no claims but we couldn't delete you, you must have legitimate children
parentQuery << name.substr(0, name.size() - 1); parentQuery << name.substr(0, name.size() - 1);
@ -275,6 +260,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
}; };
std::size_t splitPos = 0; std::size_t splitPos = 0;
auto psize = parent.size() + 1; auto psize = parent.size() + 1;
const static std::string charsThatBreakLikeOp("_%\0", 3);
for (auto& sibling: siblings) { for (auto& sibling: siblings) {
if (sibling.compare(0, psize, name, 0, psize) == 0) { if (sibling.compare(0, psize, name, 0, psize) == 0) {
splitPos = psize; splitPos = psize;
@ -294,7 +280,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
insertQuery << newNodeName << parent; insertQuery << newNodeName << parent;
insertQuery++; insertQuery++;
if (newNodeName.find('_') == std::string::npos && newNodeName.find('%') == std::string::npos) { if (newNodeName.find_first_of(charsThatBreakLikeOp) == std::string::npos) {
updateUnaffectedsQuery << newNodeName << newNodeName + "_%" << newNodeName.size(); updateUnaffectedsQuery << newNodeName << newNodeName + "_%" << newNodeName.size();
updateUnaffectedsQuery++; updateUnaffectedsQuery++;
} }
@ -308,22 +294,19 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
} }
} }
LogPrint(BCLog::CLAIMS, "Inserting or updating node %s(%s), parent %s\n", name, bin2hex(name), parent); LogPrint(BCLog::CLAIMS, "Inserting or updating node %s (%s), parent %s\n", name, HexStr(name), parent);
insertQuery << name << parent; insertQuery << name << parent;
insertQuery++; insertQuery++;
if (splitPos == 0) if (splitPos == 0)
db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent;
if (name.find('_') == std::string::npos && name.find('%') == std::string::npos) { if (name.find_first_of(charsThatBreakLikeOp) == std::string::npos) {
updateUnaffectedsQuery << name << name + "_%" << name.size(); updateUnaffectedsQuery << name << name + "_%" << name.size();
updateUnaffectedsQuery++; updateUnaffectedsQuery++;
} }
else else
db << "UPDATE nodes SET parent = ?1 WHERE SUBSTR(name, 1, ?2) = ?1 AND LENGTH(parent) < ?2 AND name != ?1" db << "UPDATE nodes SET parent = ?1 WHERE SUBSTR(name, 1, ?2) = ?1 AND LENGTH(parent) < ?2 AND name != ?1"
<< name << name.size(); << name << name.size();
if (name.size() == 1 && name[0] == 0)
db << "UPDATE nodes SET parent = '' WHERE name = ?" << name;
} }
parentQuery.used(true); parentQuery.used(true);
@ -334,7 +317,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
// parents should all be set right // parents should all be set right
db << "UPDATE nodes SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS " db << "UPDATE nodes SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS "
"(SELECT parent FROM nodes WHERE hash IS NULL UNION SELECT parent FROM prefix, nodes " "(SELECT parent FROM nodes WHERE hash IS NULL UNION SELECT parent FROM prefix, nodes "
"WHERE name = prefix.p AND prefix.p IS NOT NULL ORDER BY parent DESC) SELECT p FROM prefix)"; "WHERE name = prefix.p AND prefix.p != '') SELECT p FROM prefix)";
} }
std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const
@ -357,7 +340,7 @@ std::size_t CClaimTrieCacheBase::getTotalClaimsInTrie() const
CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) const CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) const
{ {
CAmount ret = 0; CAmount ret = 0;
std::string query("SELECT IFNULL(SUM(s.amount),0)+c.amount FROM supports s " std::string query("SELECT SUM(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM supports s "
"WHERE s.supportedClaimID = c.claimID AND s.validHeight < ?1 AND s.expirationHeight >= ?1) " "WHERE s.supportedClaimID = c.claimID AND s.validHeight < ?1 AND s.expirationHeight >= ?1) "
"FROM claims c WHERE c.validHeight < ?1 AND s.expirationHeight >= ?1"); "FROM claims c WHERE c.validHeight < ?1 AND s.expirationHeight >= ?1");
if (fControllingOnly) if (fControllingOnly)
@ -454,7 +437,7 @@ uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(const std::string& name,
if (child.hash->IsNull()) { if (child.hash->IsNull()) {
*child.hash = recursiveComputeMerkleHash(child.name, child.takeoverHeight, checkOnly); *child.hash = recursiveComputeMerkleHash(child.name, child.takeoverHeight, checkOnly);
} }
LogPrint(BCLog::CLAIMS, "Hash of %s(%s): %s, takeover: %d\n", child.name, bin2hex(child.name), (*child.hash).GetHex(), child.takeoverHeight); LogPrint(BCLog::CLAIMS, "Using hash of %s (%s): %s, takeover: %d\n", child.name, HexStr(child.name), (*child.hash).GetHex(), child.takeoverHeight);
completeHash(*child.hash, child.name, pos); completeHash(*child.hash, child.name, pos);
vchToHash.push_back(child.name[pos]); vchToHash.push_back(child.name[pos]);
vchToHash.insert(vchToHash.end(), child.hash->begin(), child.hash->end()); vchToHash.insert(vchToHash.end(), child.hash->begin(), child.hash->end());
@ -547,7 +530,7 @@ CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
nNextHeight = base->nNextHeight; nNextHeight = base->nNextHeight;
db << "PRAGMA cache_size=-" + std::to_string(200 * 1024); // in -KB db << "PRAGMA cache_size=-" + std::to_string(200 * 1024); // in -KB
db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit db << "PRAGMA synchronous=OFF"; // don't disk sync after transaction commit
db << "PRAGMA journal_mode=MEMORY"; db << "PRAGMA journal_mode=MEMORY";
db << "PRAGMA temp_store=MEMORY"; db << "PRAGMA temp_store=MEMORY";
db << "PRAGMA case_sensitive_like=true"; db << "PRAGMA case_sensitive_like=true";
@ -606,7 +589,10 @@ bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& out
db << "INSERT INTO claims(claimID, name, nodeName, txID, txN, amount, blockHeight, validHeight, expirationHeight, metadata) " db << "INSERT INTO claims(claimID, name, nodeName, txID, txN, amount, blockHeight, validHeight, expirationHeight, metadata) "
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << claimId << name << nodeName "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << claimId << name << nodeName
<< outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << expires << metadata; << outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << expires << metadata;
if (nValidHeight < nNextHeight)
db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName;
return true; return true;
} }
@ -623,7 +609,9 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << supportedClaimId << name << nodeName "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << supportedClaimId << name << nodeName
<< outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << expires << metadata; << outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << expires << metadata;
db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; if (nValidHeight < nNextHeight)
db << "UPDATE nodes SET hash = NULL WHERE name = ?" << nodeName;
return true; return true;
} }
@ -961,10 +949,11 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimUndoTy
// for all dirty nodes look for new takeovers // for all dirty nodes look for new takeovers
if (!transacting) { transacting = true; db << "begin"; } if (!transacting) { transacting = true; db << "begin"; }
db << "INSERT INTO nodes(name) SELECT nodeName FROM claims "
"WHERE validHeight = ?1 AND expirationHeight > ?1 "
"ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight;
db << "UPDATE nodes SET hash = NULL WHERE name IN " db << "UPDATE nodes SET hash = NULL WHERE name IN "
"(SELECT nodeName FROM claims WHERE validHeight = ?)" << nNextHeight; "(SELECT nodeName FROM supports WHERE validHeight = ?1 AND expirationHeight > ?1)" << nNextHeight;
db << "UPDATE nodes SET hash = NULL WHERE name IN "
"(SELECT nodeName FROM supports WHERE validHeight = ?)" << nNextHeight;
assert(expireUndo.empty()); assert(expireUndo.empty());
{ {
@ -1027,7 +1016,7 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimUndoTy
if (takeoverHappening && activateAllFor(insertUndo, insertSupportUndo, nameWithTakeover)) if (takeoverHappening && activateAllFor(insertUndo, insertSupportUndo, nameWithTakeover))
hasCandidate = getInfoForName(nameWithTakeover, candidateValue, 1); hasCandidate = getInfoForName(nameWithTakeover, candidateValue, 1);
LogPrint(BCLog::CLAIMS, "Takeover on %s(%s) at %d, happening: %d, set before: %d\n", nameWithTakeover, bin2hex(nameWithTakeover), nNextHeight, takeoverHappening, hasBeenSetBefore); LogPrint(BCLog::CLAIMS, "Takeover on %s (%s) at %d, happening: %d, set before: %d\n", nameWithTakeover, HexStr(nameWithTakeover), nNextHeight, takeoverHappening, hasBeenSetBefore);
if (takeoverHappening || !hasBeenSetBefore) { if (takeoverHappening || !hasBeenSetBefore) {
takeoverUndo.emplace_back(nameWithTakeover, std::make_pair(existingHeight, hasBeenSetBefore ? *existingID : uint160())); takeoverUndo.emplace_back(nameWithTakeover, std::make_pair(existingHeight, hasBeenSetBefore ? *existingID : uint160()));
@ -1129,9 +1118,9 @@ bool CClaimTrieCacheBase::decrementBlock(insertUndoType& insertUndo, claimUndoTy
bool CClaimTrieCacheBase::finalizeDecrement(takeoverUndoType& takeoverUndo) bool CClaimTrieCacheBase::finalizeDecrement(takeoverUndoType& takeoverUndo)
{ {
db << "UPDATE nodes SET hash = NULL WHERE name IN " db << "UPDATE nodes SET hash = NULL WHERE name IN "
"(SELECT nodeName FROM claims WHERE validHeight = ?)" << nNextHeight; "(SELECT nodeName FROM claims WHERE validHeight = ?1 AND expirationHeight > ?1)" << nNextHeight;
db << "UPDATE nodes SET hash = NULL WHERE name IN " db << "UPDATE nodes SET hash = NULL WHERE name IN "
"(SELECT nodeName FROM supports WHERE validHeight = ?)" << nNextHeight; "(SELECT nodeName FROM supports WHERE validHeight = ?1 AND expirationHeight > ?1)" << nNextHeight;
for (auto it = takeoverUndo.crbegin(); it != takeoverUndo.crend(); ++it) { for (auto it = takeoverUndo.crbegin(); it != takeoverUndo.crend(); ++it) {
if (it->second.second.IsNull()) if (it->second.second.IsNull())

2
src/claimtrie.h
View file

@ -392,7 +392,7 @@ protected:
virtual int getDelayForName(const std::string& name, const uint160& claimId) const; virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
bool deleteNodeIfPossible(const std::string& name, std::string& parent, std::vector<std::string>& claims); bool deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims);
void ensureTreeStructureIsUpToDate(); void ensureTreeStructureIsUpToDate();
private: private:

72
src/test/claimtriehashfork_tests.cpp
View file

@ -401,4 +401,76 @@ BOOST_AUTO_TEST_CASE(bogus_claimtrie_hash_test)
BOOST_CHECK_EQUAL(orig_chain_height, chainActive.Height()); BOOST_CHECK_EQUAL(orig_chain_height, chainActive.Height());
} }
#ifndef MAC_OSX // can't find a random number generator that produces the same sequence on OSX
BOOST_AUTO_TEST_CASE(triehash_fuzzer_test)
{
ClaimTrieChainFixture fixture;
auto envClaims = std::getenv("TRIEHASH_FUZZER_CLAIMS");
auto envBlocks = std::getenv("TRIEHASH_FUZZER_BLOCKS");
const int claimsPerBlock = envClaims ? std::atoi(envClaims) : 100;
const int blocks = envBlocks ? std::atoi(envBlocks) : 13;
auto names = random_strings(blocks * claimsPerBlock);
FastRandomContext frc(true);
std::unordered_map<std::string, std::vector<CMutableTransaction>> existingClaims;
std::vector<CMutableTransaction> existingSupports;
std::string value(1024, 'c');
std::vector<CTransaction> cb {fixture.GetCoinbase()};
for (int i = 0; i < blocks; ++i) {
for (int j = 0; j < claimsPerBlock; ++j) {
auto name = names[i * claimsPerBlock + j];
auto supportFront = frc.randrange(4) == 0;
auto supportBack = frc.randrange(4) == 0;
auto removeClaim = frc.randrange(4) == 0;
auto removeSupport = frc.randrange(4) == 0;
auto hit = existingClaims.find(name);
if (supportFront && hit != existingClaims.end() && hit->second.size()) {
auto tx = fixture.MakeSupport(cb.back(), hit->second[frc.rand64() % hit->second.size()], name, 2);
existingSupports.push_back(tx);
cb.emplace_back(std::move(tx));
}
if (removeClaim && hit != existingClaims.end() && hit->second.size()) {
auto idx = frc.rand64() % hit->second.size();
fixture.Spend(hit->second[idx]);
hit->second.erase(hit->second.begin() + idx);
} else {
auto tx = fixture.MakeClaim(cb.back(), name, value, 2);
existingClaims[name].push_back(tx);
hit = existingClaims.find(name);
cb.emplace_back(std::move(tx));
}
if (supportBack && hit != existingClaims.end() && hit->second.size()) {
auto tx = fixture.MakeSupport(cb.back(), hit->second[frc.rand64() % hit->second.size()], name, 2);
existingSupports.push_back(tx);
cb.emplace_back(std::move(tx));
}
if (removeSupport && (i & 7) == 7 && !existingSupports.empty()) {
const auto tidx = frc.rand64() % existingSupports.size();
const auto tx = existingSupports[tidx];
fixture.Spend(tx);
existingSupports.erase(existingSupports.begin() + tidx);
}
if (cb.back().GetValueOut() < 10 || cb.size() > 40000) {
cb.clear();
cb.push_back(fixture.GetCoinbase());
}
}
fixture.IncrementBlocks(1);
if (blocks > 13 && i % 50 == 0) // travisCI needs some periodic output
std::cerr << "In triehash_fuzzer_test with " << fixture.getTotalNamesInTrie() << " names at block " << i << std::endl;
}
if (blocks == 1000 && claimsPerBlock == 100)
BOOST_CHECK_EQUAL(fixture.getMerkleHash().GetHex(), "28825257a129eef69cab87d6255c8359fc6dc083ca7f09222526e3a7971f382d");
else if (blocks == 13 && claimsPerBlock == 100)
BOOST_CHECK_EQUAL(fixture.getMerkleHash().GetHex(), "4e5984d6984f5f05d50e821e6228d56bcfbd16ca2093cd0308f6ff1c2bc8689a");
else
std::cerr << "Hash: " << fixture.getMerkleHash().GetHex() << std::endl;
}
#endif
BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END()