From cc32e2949cc392c1524dd0e6a21e96e9bce6c8f8 Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Mon, 9 Dec 2019 14:58:16 +0200 Subject: [PATCH] Address performance issue Signed-off-by: Anthony Fieroni --- src/claimtrie/forks.cpp | 4 +- .../sqlite/hdr/sqlite_modern_cpp/errors.h | 4 +- src/claimtrie/trie.cpp | 147 +++++++++--------- src/claimtrie/trie.h | 3 +- 4 files changed, 77 insertions(+), 81 deletions(-) diff --git a/src/claimtrie/forks.cpp b/src/claimtrie/forks.cpp index ed146ff00..d2010662a 100644 --- a/src/claimtrie/forks.cpp +++ b/src/claimtrie/forks.cpp @@ -330,6 +330,8 @@ std::vector ComputeMerklePath(const std::vector& hashes, uin return res; } +extern const std::string proofClaimQuery_s; + bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const CUint160& claim, CClaimTrieProof& proof) { if (nNextHeight < base->nAllClaimsInMerkleForkHeight) @@ -344,7 +346,7 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const CUi // cache the parent nodes getMerkleHash(); proof = CClaimTrieProof(); - for (auto&& row: db << proofClaimQuery << name) { + for (auto&& row: db << proofClaimQuery_s << name) { std::string key; int takeoverHeight; row >> key >> takeoverHeight; diff --git a/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/errors.h b/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/errors.h index 07e1b1fa8..c68158893 100644 --- a/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/errors.h +++ b/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/errors.h @@ -9,8 +9,8 @@ namespace sqlite { class sqlite_exception: public std::runtime_error { public: - sqlite_exception(const char* msg, str_ref sql, int code = -1): runtime_error(msg), code(code), sql(sql) {} - sqlite_exception(int code, str_ref sql, const char *msg = nullptr): runtime_error(msg ? msg : sqlite3_errstr(code)), code(code), sql(sql) {} + sqlite_exception(const char* msg, str_ref sql, int code = -1): runtime_error(sql + ':' + msg), code(code), sql(sql) {} + sqlite_exception(int code, str_ref sql, const char *msg = nullptr): runtime_error(sql + ':' + (msg ? msg : sqlite3_errstr(code))), code(code), sql(sql) {} int get_code() const {return code & 0xFF;} int get_extended_code() const {return code;} std::string get_sql() const {return sql;} diff --git a/src/claimtrie/trie.cpp b/src/claimtrie/trie.cpp index 6fec27136..578080072 100644 --- a/src/claimtrie/trie.cpp +++ b/src/claimtrie/trie.cpp @@ -115,6 +115,7 @@ CClaimTrieCacheBase::~CClaimTrieCacheBase() } claimHashQuery.used(true); childHashQuery.used(true); + claimHashQueryLimit.used(true); } bool CClaimTrie::SyncToDisk() @@ -247,6 +248,9 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() auto insertQuery = db << "INSERT INTO nodes(name, parent, hash) VALUES(?, ?, NULL) " "ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL"; + auto nodesQuery = db << "SELECT name FROM nodes WHERE parent = ? ORDER BY name"; + auto updateQuery = db << "UPDATE nodes SET parent = ? WHERE name = ?"; + for (auto& name: names) { int64_t claims; std::string parent, node; @@ -265,42 +269,38 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() // we know now that we need to insert it, // but we may need to insert a parent node for it first (also called a split) - std::vector siblings; - db << "SELECT name FROM nodes WHERE parent = ?" << parent - >> [&siblings](std::string name) { - siblings.push_back(std::move(name)); - }; - std::sort(siblings.begin(), siblings.end()); // not strictly necessary but helps with determinism in debugging - - std::size_t splitPos = 0; - auto psize = parent.size() + 1; - for (auto& sibling: siblings) { - if (sibling.compare(0, psize, name, 0, psize) == 0) { - splitPos = psize; - while(splitPos < sibling.size() && splitPos < name.size() && sibling[splitPos] == name[splitPos]) - ++splitPos; - auto newNodeName = name.substr(0, splitPos); - // update the to-be-fostered sibling: - db << "UPDATE nodes SET parent = ? WHERE name = ?" << newNodeName << sibling; - if (splitPos == name.size()) { - // our new node is the same as the one we wanted to insert - break; - } - // insert the split node: - logPrint << "Inserting split node " << newNodeName << " near " << sibling << ", parent " << parent << Clog::endl; - insertQuery << newNodeName << parent; - insertQuery++; - - parent = std::move(newNodeName); + const auto psize = parent.size() + 1; + for (auto&& row : nodesQuery << parent) { + std::string sibling; row >> sibling; + if (sibling.compare(0, psize, name, 0, psize) != 0) + continue; + auto splitPos = psize; + while(splitPos < sibling.size() && splitPos < name.size() && sibling[splitPos] == name[splitPos]) + ++splitPos; + auto newNodeName = name.substr(0, splitPos); + // update the to-be-fostered sibling: + updateQuery << newNodeName << sibling; + updateQuery++; + if (splitPos == name.size()) + // our new node is the same as the one we wanted to insert break; - } + // insert the split node: + logPrint << "Inserting split node " << newNodeName << " near " << sibling << ", parent " << parent << Clog::endl; + insertQuery << newNodeName << parent; + insertQuery++; + + parent = std::move(newNodeName); + break; } + nodesQuery++; logPrint << "Inserting or updating node " << name << ", parent " << parent << Clog::endl; insertQuery << name << parent; insertQuery++; } + nodesQuery.used(true); + updateQuery.used(true); parentQuery.used(true); insertQuery.used(true); @@ -340,7 +340,7 @@ int64_t CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) "WHERE c.activationHeight < ?1 AND c.expirationHeight >= ?1 GROUP BY c.nodeName)" : "SELECT SUM(amount) FROM (SELECT c.amount as amount " - "FROM claims c WHERE c.activationHeight < ?1 AND s.expirationHeight >= ?1)"; + "FROM claims c WHERE c.activationHeight < ?1 AND c.expirationHeight >= ?1)"; db << query << nNextHeight >> ret; return ret; @@ -350,38 +350,21 @@ bool CClaimTrieCacheBase::getInfoForName(const std::string& name, CClaimValue& c { auto ret = false; auto nextHeight = nNextHeight + heightOffset; - for (auto&& row: claimHashQuery << nextHeight << name) { + for (auto&& row: claimHashQueryLimit << nextHeight << name) { row >> claim.outPoint.hash >> claim.outPoint.n >> claim.claimId >> claim.nHeight >> claim.nValidAtHeight >> claim.nAmount >> claim.nEffectiveAmount; ret = true; break; } - claimHashQuery++; + claimHashQueryLimit++; return ret; } CClaimSupportToName CClaimTrieCacheBase::getClaimsForName(const std::string& name) const { + CUint160 claimId; int nLastTakeoverHeight = 0; - { - auto query = db << "SELECT CASE WHEN claimID IS NULL THEN 0 ELSE height END " - "FROM takeover WHERE name = ? ORDER BY height DESC LIMIT 1" << name; - for (auto&& row: query) { - row >> nLastTakeoverHeight; - } - } - claimEntryType claims; - { - auto query = db << "SELECT claimID, txID, txN, blockHeight, activationHeight, amount " - "FROM claims WHERE nodeName = ? AND expirationHeight >= ?" - << name << nNextHeight; - for (auto &&row: query) { - CClaimValue claim; - row >> claim.claimId >> claim.outPoint.hash >> claim.outPoint.n - >> claim.nHeight >> claim.nValidAtHeight >> claim.nAmount; - claims.push_back(std::move(claim)); - } - } + getLastTakeoverForName(name, claimId, nLastTakeoverHeight); auto supports = getSupportsForName(name); auto find = [&supports](decltype(supports)::iterator& it, const CClaimValue& claim) { @@ -391,15 +374,22 @@ CClaimSupportToName CClaimTrieCacheBase::getClaimsForName(const std::string& nam return it != supports.end(); }; + auto query = db << "SELECT claimID, txID, txN, blockHeight, activationHeight, amount " + "FROM claims WHERE nodeName = ? AND expirationHeight >= ?" + << name << nNextHeight; + // match support to claim std::vector claimsNsupports; - for (const auto& claim : claims) { + for (auto &&row: query) { + CClaimValue claim; + row >> claim.claimId >> claim.outPoint.hash >> claim.outPoint.n + >> claim.nHeight >> claim.nValidAtHeight >> claim.nAmount; int64_t nAmount = claim.nValidAtHeight < nNextHeight ? claim.nAmount : 0; auto ic = claimsNsupports.emplace(claimsNsupports.end(), claim, nAmount); for (auto it = supports.begin(); find(it, claim); it = supports.erase(it)) { if (it->nValidAtHeight < nNextHeight) ic->effectiveAmount += it->nAmount; - ic->supports.emplace_back(*it); + ic->supports.push_back(std::move(*it)); } ic->claim.nEffectiveAmount = ic->effectiveAmount; } @@ -498,23 +488,30 @@ bool CClaimTrieCacheBase::flush() return true; } +const std::string childHashQuery_s = "SELECT name, hash FROM nodes WHERE parent = ? ORDER BY name"; + +const std::string claimHashQuery_s = + "SELECT c.txID, c.txN, c.claimID, c.blockHeight, c.activationHeight, c.amount, " + "(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM supports s " + "WHERE s.supportedClaimID = c.claimID AND s.nodeName = c.nodeName " + "AND s.activationHeight < ?1 AND s.expirationHeight >= ?1) as effectiveAmount " + "FROM claims c WHERE c.nodeName = ?2 AND c.activationHeight < ?1 AND c.expirationHeight >= ?1 " + "ORDER BY effectiveAmount DESC, c.blockHeight, c.txID, c.txN"; + +const std::string claimHashQueryLimit_s = claimHashQuery_s + " LIMIT 1"; + +extern const std::string proofClaimQuery_s = + "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END " + "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM nodes n " + "WHERE n.name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " + "SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix) " + "ORDER BY n.name"; + CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) : base(base), db(base->dbFile, sharedConfig), transacting(false), - childHashQuery(db << "SELECT n.name, n.hash, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END FROM takeover t " - "WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM nodes n " - "WHERE n.parent = ? ORDER BY n.name"), - claimHashQuery(db << "SELECT c.txID, c.txN, c.claimID, c.blockHeight, c.activationHeight, c.amount, " - "(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM supports s " - "WHERE s.supportedClaimID = c.claimID AND s.nodeName = c.nodeName " - "AND s.activationHeight < ?1 AND s.expirationHeight >= ?1) as effectiveAmount " - "FROM claims c WHERE c.nodeName = ?2 AND c.activationHeight < ?1 " - "AND c.expirationHeight >= ?1 " - "ORDER BY effectiveAmount DESC, c.blockHeight, c.txID, c.txN"), - proofClaimQuery("SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END " - "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM nodes n " - "WHERE n.name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " - "SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix) " - "ORDER BY n.name") + childHashQuery(db << childHashQuery_s), + claimHashQuery(db << claimHashQuery_s), + claimHashQueryLimit(db << claimHashQueryLimit_s) { assert(base); nNextHeight = base->nNextHeight; @@ -1165,9 +1162,8 @@ bool CClaimTrieCacheBase::incrementBlock() int existingHeight = 0; auto hasCurrentWinner = getLastTakeoverForName(nameWithTakeover, existingID, existingHeight); // we have a takeover if we had a winner and its changing or we never had a winner - auto takeoverHappening = hasCandidate && hasCurrentWinner && existingID != candidateValue.claimId; - takeoverHappening |= !hasCandidate && hasCurrentWinner; - takeoverHappening |= hasCandidate && !hasCurrentWinner; + auto takeoverHappening = (hasCandidate && !hasCurrentWinner) || (hasCurrentWinner && existingID != candidateValue.claimId); + if (takeoverHappening && activateAllFor(nameWithTakeover)) hasCandidate = getInfoForName(nameWithTakeover, candidateValue, 1); @@ -1276,7 +1272,7 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, const CUint16 // cache the parent nodes getMerkleHash(); proof = CClaimTrieProof(); - for (auto&& row: db << proofClaimQuery << name) { + for (auto&& row: db << proofClaimQuery_s << name) { CClaimValue claim; std::string key; int takeoverHeight; @@ -1324,8 +1320,8 @@ bool CClaimTrieCacheBase::findNameForClaim(std::vector claim, CCl { std::reverse(claim.begin(), claim.end()); auto query = db << "SELECT nodeName, claimID, txID, txN, amount, activationHeight, blockHeight " - "FROM claims WHERE SUBSTR(claimID, ?) = ? AND activationHeight < ? AND expirationHeight >= ?" - << -int(claim.size()) << claim << nNextHeight << nNextHeight; + "FROM claims WHERE SUBSTR(claimID, ?1) = ?2 AND activationHeight < ?3 AND expirationHeight >= ?3" + << -int(claim.size()) << claim << nNextHeight; auto hit = false; for (auto&& row: query) { if (hit) return false; @@ -1338,9 +1334,8 @@ bool CClaimTrieCacheBase::findNameForClaim(std::vector claim, CCl void CClaimTrieCacheBase::getNamesInTrie(std::function callback) const { - db << "SELECT DISTINCT nodeName FROM claims WHERE activationHeight < ? AND expirationHeight >= ?" - << nNextHeight << nNextHeight - >> [&callback](const std::string& name) { + db << "SELECT DISTINCT nodeName FROM claims WHERE activationHeight < ?1 AND expirationHeight >= ?1" + << nNextHeight >> [&callback](const std::string& name) { callback(name); }; } diff --git a/src/claimtrie/trie.h b/src/claimtrie/trie.h index 82e569d22..ea085cbb3 100644 --- a/src/claimtrie/trie.h +++ b/src/claimtrie/trie.h @@ -106,10 +106,9 @@ protected: int nNextHeight; // Height of the block that is being worked on, which is CClaimTrie* base; mutable sqlite::database db; - const std::string proofClaimQuery; mutable std::unordered_set removalWorkaround; - mutable sqlite::database_binder claimHashQuery, childHashQuery; + mutable sqlite::database_binder claimHashQuery, childHashQuery, claimHashQueryLimit; virtual CUint256 computeNodeHash(const std::string& name, int takeoverHeight); supportEntryType getSupportsForName(const std::string& name) const;