LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Merkle hash improve #390

Merged
bvbfan merged 2 commits from merkle_improve into v17_master 2020-06-16 20:23:54 +02:00
Conversation 0 Commits 2 Files changed 7 +123 -61
6 changed files with 123 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit e66c2aea32 - Show all commits

34
src/claimtrie/forks.cpp
View file

@ -139,10 +139,10 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary()
// make the new nodes
db << "INSERT INTO node(name) SELECT NORMALIZED(name) AS nn FROM claim WHERE nn != nodeName "
"AND activationHeight <= ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight;
"AND activationHeight <= ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight;
// there's a subtlety here: names in supports don't make new nodes
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT NORMALIZED(name) AS nn FROM support WHERE nn != nodeName "
"AND activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight;
@ -151,7 +151,7 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary()
db << "UPDATE support SET nodeName = NORMALIZED(name) WHERE activationHeight <= ?1 AND expirationHeight > ?1" << nNextHeight;
// remove the old nodes
db << "UPDATE node SET hash = NULL WHERE name NOT IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN "
"(SELECT nodeName FROM claim WHERE activationHeight <= ?1 AND expirationHeight > ?1 "
"UNION SELECT nodeName FROM support WHERE activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight;
@ -167,9 +167,9 @@ bool CClaimTrieCacheNormalizationFork::unnormalizeAllNamesInTrieIfNecessary()
ensureTransacting();
db << "INSERT INTO node(name) SELECT name FROM claim WHERE name != nodeName "
"AND activationHeight < ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight;
"AND activationHeight < ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight;
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT nodeName FROM support WHERE name != nodeName "
"UNION SELECT nodeName FROM claim WHERE name != nodeName)";
@ -177,7 +177,7 @@ bool CClaimTrieCacheNormalizationFork::unnormalizeAllNamesInTrieIfNecessary()
db << "UPDATE support SET nodeName = name";
// we need to let the tree structure method do the actual node delete
db << "UPDATE node SET hash = NULL WHERE name NOT IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN "
"(SELECT DISTINCT name FROM claim)";
return true;
@ -242,10 +242,10 @@ uint256 ComputeMerkleRoot(std::vector<uint256> hashes)
return hashes.empty() ? uint256{} : hashes[0];
}
uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, int takeoverHeight)
uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight)
{
if (nNextHeight < base->nAllClaimsInMerkleForkHeight)
return CClaimTrieCacheNormalizationFork::computeNodeHash(name, takeoverHeight);
return CClaimTrieCacheNormalizationFork::computeNodeHash(name, claimsHash, takeoverHeight);
std::vector<uint256> childHashes;
childHashQuery << name >> [&childHashes](std::string, uint256 hash) {
@ -253,20 +253,24 @@ uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, int ta
};
childHashQuery++;
std::vector<uint256> claimHashes;
if (takeoverHeight > 0) {
if (claimsHash.IsNull()) {
COutPoint p;
std::vector<uint256> hashes;
for (auto&& row: claimHashQuery << nNextHeight << name) {
row >> p.hash >> p.n;
claimHashes.push_back(getValueHash(p, takeoverHeight));
hashes.push_back(getValueHash(p, takeoverHeight));
}
claimHashQuery++;
claimsHash = hashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(hashes));
}
} else {
claimsHash = emptyHash;
}
auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(claimHashes));
const auto& childrenHash = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
return Hash(left.begin(), left.end(), right.begin(), right.end());
return Hash(childrenHash.begin(), childrenHash.end(), claimsHash.begin(), claimsHash.end());
}
std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
@ -397,7 +401,7 @@ void CClaimTrieCacheHashFork::initializeIncrement()
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
if (nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) {
ensureTransacting();
db << "UPDATE node SET hash = NULL";
db << "UPDATE node SET hash = NULL, claimsHash = NULL";
}
}
@ -406,7 +410,7 @@ bool CClaimTrieCacheHashFork::finalizeDecrement()
auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement();
if (ret && nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) {
ensureTransacting();
db << "UPDATE node SET hash = NULL";
db << "UPDATE node SET hash = NULL, claimsHash = NULL";
}
return ret;
}

2
src/claimtrie/forks.h
View file

@ -62,7 +62,7 @@ public:
bool allowSupportMetadata() const;
protected:
uint256 computeNodeHash(const std::string& name, int takeoverHeight) override;
uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight) override;
};
typedef CClaimTrieCacheHashFork CClaimTrieCache;

117
src/claimtrie/trie.cpp
View file

@ -76,7 +76,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
db << "CREATE TABLE IF NOT EXISTS node (name BLOB NOT NULL PRIMARY KEY, "
"parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"hash BLOB)";
"hash BLOB, claimsHash BLOB)";
db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL PRIMARY KEY, name BLOB NOT NULL, "
"nodeName BLOB NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
@ -99,6 +99,8 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
db << "DELETE FROM takeover";
}
doNodeTableMigration();
db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)";
// db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain
db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)";
@ -117,6 +119,29 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
db << "INSERT OR IGNORE INTO node(name, hash) VALUES(x'', ?)" << one; // ensure that we always have our root node
}
void CClaimTrie::doNodeTableMigration()
{
try {
isNodeMigrationStart = false;
for (auto&& row : db << "SELECT claimsHash FROM node WHERE name = x''")
break;
} catch (const sqlite::sqlite_exception&) {
isNodeMigrationStart = true;
// new node schema
db << "CREATE TABLE node_new (name BLOB NOT NULL PRIMARY KEY, "
"parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"hash BLOB, claimsHash BLOB)";
db << "INSERT OR REPLACE INTO node_new(name, parent, hash) "
"SELECT name, parent, hash FROM node";
db << "DROP TABLE node";
db << "ALTER TABLE node_new RENAME TO node";
}
}
CClaimTrieCacheBase::~CClaimTrieCacheBase()
{
if (transacting) {
@ -277,7 +302,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate()
"name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
"SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix)";
auto insertQuery = db << "INSERT INTO node(name, parent, hash) VALUES(?, ?, NULL) "
auto insertQuery = db << "INSERT INTO node(name, parent, hash, claimsHash) VALUES(?, ?, NULL, NULL) "
"ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL";
auto nodeQuery = db << "SELECT name FROM node WHERE parent = ?";
@ -436,7 +461,7 @@ void completeHash(uint256& partialHash, const std::string& key, int to)
partialHash = Hash(it, it + 1, partialHash.begin(), partialHash.end());
}
uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, int takeoverHeight)
uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight)
{
const auto pos = name.size();
std::vector<uint8_t> vchToHash;
@ -449,11 +474,13 @@ uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, int takeov
childHashQuery++;
if (takeoverHeight > 0) {
if (claimsHash.IsNull()) {
CClaimValue claim;
if (getInfoForName(name, claim)) {
auto valueHash = getValueHash(claim.outPoint, takeoverHeight);
vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
if (getInfoForName(name, claim))
claimsHash = getValueHash(claim.outPoint, takeoverHeight);
}
if (!claimsHash.IsNull())
vchToHash.insert(vchToHash.end(), claimsHash.begin(), claimsHash.end());
}
return vchToHash.empty() ? one : Hash(vchToHash.begin(), vchToHash.end());
@ -471,22 +498,47 @@ bool CClaimTrieCacheBase::checkConsistency()
}
}
if (base->isNodeMigrationStart)
ensureTransacting();
auto updateQuery = db << "UPDATE node SET claimsHash = ? WHERE name = ?";
// not checking everything as it takes too long
auto query = db << "SELECT n.name, n.hash, "
auto query = db << (base->isNodeMigrationStart ?
"SELECT n.name, n.hash, n.claimsHash, "
"IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM node n "
"WHERE n.name IN (SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 100000) OR n.parent = x''";
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) "
"FROM node n ORDER BY LENGTH(n.name) DESC"
:
"SELECT n.name, n.hash, n.claimsHash, "
"IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) "
"FROM node n WHERE n.name IN "
"(SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 100000) OR n.parent = x''");
for (auto&& row: query) {
std::string name;
uint256 hash;
int takeoverHeight;
row >> name >> hash >> takeoverHeight;
auto computedHash = computeNodeHash(name, takeoverHeight);
uint256 hash, claimsHash, computedClaimsHash;
row >> name >> hash >> claimsHash >> takeoverHeight;
auto computedHash = computeNodeHash(name, computedClaimsHash, takeoverHeight);
if (computedHash != hash) {
logPrint << "Invalid hash at " << name << Clog::endl;
return false;
}
if (base->isNodeMigrationStart) {
assert(!computedClaimsHash.IsNull());
updateQuery << computedClaimsHash << name;
updateQuery++;
} else if (computedClaimsHash != claimsHash) {
logPrint << "Invalid claimsHash at " << name << Clog::endl;
return false;
}
}
updateQuery.used(true);
if (base->isNodeMigrationStart)
return flush();
return true;
}
@ -572,20 +624,20 @@ uint256 CClaimTrieCacheBase::getMerkleHash()
{
ensureTreeStructureIsUpToDate();
uint256 hash;
db << "SELECT hash FROM node WHERE name = x''"
>> [&hash](std::unique_ptr<uint256> rootHash) {
if (rootHash)
hash = std::move(*rootHash);
};
for (auto&& row : db << "SELECT hash FROM node WHERE name = x''") {
row >> hash;
if (!hash.IsNull())
return hash;
}
assert(transacting); // no data changed but we didn't have the root hash there already?
auto updateQuery = db << "UPDATE node SET hash = ? WHERE name = ?";
db << "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END FROM takeover t WHERE t.name = n.name "
"ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob
>> [this, &hash, &updateQuery](const std::string& name, int takeoverHeight) {
hash = computeNodeHash(name, takeoverHeight);
updateQuery << hash << name;
auto updateQuery = db << "UPDATE node SET hash = ?, claimsHash = ? WHERE name = ?";
db << "SELECT n.name, n.claimsHash, "
"IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) "
"FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob
>> [&](const std::string& name, uint256 claimsHash, int takeoverHeight) {
hash = computeNodeHash(name, claimsHash, takeoverHeight);
updateQuery << hash << claimsHash << name;
updateQuery++;
};
updateQuery.used(true);
@ -639,7 +691,7 @@ bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& out
<< originalHeight << nHeight << nValidHeight << nValidHeight << expires;
if (nValidHeight < nNextHeight)
db << "INSERT INTO node(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName;
db << "INSERT INTO node(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nodeName;
return true;
}
@ -660,7 +712,7 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o
<< supportedClaimId << name << nodeName << outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << nValidHeight << expires;
if (nValidHeight < nNextHeight)
db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName;
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName;
return true;
}
@ -691,7 +743,7 @@ bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, const COutPoint& o
if (!db.rows_modified())
return false;
db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName;
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName;
// when node should be deleted from cache but instead it's kept
// because it's a parent one and should not be effectively erased
@ -722,7 +774,8 @@ bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string&
db << "DELETE FROM support WHERE txID = ? AND txN = ?" << outPoint.hash << outPoint.n;
if (!db.rows_modified())
return false;
db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName;
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName;
return true;
}
@ -739,11 +792,11 @@ bool CClaimTrieCacheBase::incrementBlock()
db << "INSERT INTO node(name) SELECT nodeName FROM claim INDEXED BY claim_activationHeight "
"WHERE activationHeight = ?1 AND expirationHeight > ?1 "
"ON CONFLICT(name) DO UPDATE SET hash = NULL"
"ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL"
<< nNextHeight;
// don't make new nodes for items in supports or items that expire this block that don't exist in claims
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT nodeName FROM claim WHERE expirationHeight = ?1 "
"UNION SELECT nodeName FROM support WHERE expirationHeight = ?1 OR activationHeight = ?1)"
<< nNextHeight;
@ -818,10 +871,10 @@ bool CClaimTrieCacheBase::decrementBlock()
nNextHeight--;
db << "INSERT INTO node(name) SELECT nodeName FROM claim "
"WHERE expirationHeight = ? ON CONFLICT(name) DO UPDATE SET hash = NULL"
"WHERE expirationHeight = ? ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL"
<< nNextHeight;
db << "UPDATE node SET hash = NULL WHERE name IN("
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN("
"SELECT nodeName FROM support WHERE expirationHeight = ?1 OR activationHeight = ?1 "
"UNION SELECT nodeName FROM claim WHERE activationHeight = ?1)"
<< nNextHeight;
@ -837,7 +890,7 @@ bool CClaimTrieCacheBase::decrementBlock()
bool CClaimTrieCacheBase::finalizeDecrement()
{
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT nodeName FROM claim WHERE activationHeight = ?1 AND expirationHeight > ?1 "
"UNION SELECT nodeName FROM support WHERE activationHeight = ?1 AND expirationHeight > ?1 "
"UNION SELECT name FROM takeover WHERE height = ?1)" << nNextHeight;

6
src/claimtrie/trie.h
View file

@ -49,6 +49,7 @@ protected:
std::size_t dbCacheBytes;
const std::string dbFile;
sqlite::database db;
bool isNodeMigrationStart;
const int nProportionalDelayFactor;
const int nNormalizedNameForkHeight;
@ -57,6 +58,9 @@ protected:
const int64_t nExtendedClaimExpirationTime;
const int64_t nExtendedClaimExpirationForkHeight;
const int64_t nAllClaimsInMerkleForkHeight;
private:
void doNodeTableMigration();
};
class CClaimTrieCacheBase
@ -119,7 +123,7 @@ protected:
mutable sqlite::database_binder claimHashQuery, childHashQuery, claimHashQueryLimit;
virtual uint256 computeNodeHash(const std::string& name, int takeoverHeight);
virtual uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight);
supportEntryType getSupportsForName(const std::string& name) const;
virtual int getDelayForName(const std::string& name, const uint160& claimId) const;

1
src/httpserver.cpp
View file

@ -13,6 +13,7 @@
#include <sync.h>
#include <ui_interface.h>
#include <deque>
#include <memory>
#include <stdio.h>
#include <stdlib.h>

10
src/validation.cpp
View file

@ -2059,15 +2059,15 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
// TODO: if the "just check" flag is set, we should reduce the work done here. Incrementing blocks twice per mine is not efficient.
assert(trieCache.incrementBlock());
int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2;
LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal);
if (trieCache.getMerkleHash() != block.hashClaimTrie) {
return state.DoS(100, error("ConnectBlock() : the merkle root of the claim trie does not match "
"(actual=%s vs block=%s on height=%d)", trieCache.getMerkleHash().GetHex(),
block.hashClaimTrie.GetHex(), pindex->nHeight), REJECT_INVALID, "bad-claim-merkle-hash");
}
int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2;
LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal);
CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus());
if (block.vtx[0]->GetValueOut() > blockReward)
return state.DoS(100,
@ -2077,8 +2077,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
if (!control.Wait())
return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed");
int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2;
LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal);
int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime3;
LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime3), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime3) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal);
if (fJustCheck)
return true;