diff --git a/src/claimtrie/forks.cpp b/src/claimtrie/forks.cpp index ed7898139..66af212dc 100644 --- a/src/claimtrie/forks.cpp +++ b/src/claimtrie/forks.cpp @@ -139,10 +139,10 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary() // make the new nodes db << "INSERT INTO node(name) SELECT NORMALIZED(name) AS nn FROM claim WHERE nn != nodeName " - "AND activationHeight <= ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight; + "AND activationHeight <= ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight; // there's a subtlety here: names in supports don't make new nodes - db << "UPDATE node SET hash = NULL WHERE name IN " + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN " "(SELECT NORMALIZED(name) AS nn FROM support WHERE nn != nodeName " "AND activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight; @@ -151,7 +151,7 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary() db << "UPDATE support SET nodeName = NORMALIZED(name) WHERE activationHeight <= ?1 AND expirationHeight > ?1" << nNextHeight; // remove the old nodes - db << "UPDATE node SET hash = NULL WHERE name NOT IN " + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN " "(SELECT nodeName FROM claim WHERE activationHeight <= ?1 AND expirationHeight > ?1 " "UNION SELECT nodeName FROM support WHERE activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight; @@ -167,9 +167,9 @@ bool CClaimTrieCacheNormalizationFork::unnormalizeAllNamesInTrieIfNecessary() ensureTransacting(); db << "INSERT INTO node(name) SELECT name FROM claim WHERE name != nodeName " - "AND activationHeight < ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight; + "AND activationHeight < ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight; - db << "UPDATE node SET hash = NULL WHERE name IN " + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN " "(SELECT nodeName FROM support WHERE name != nodeName " "UNION SELECT nodeName FROM claim WHERE name != nodeName)"; @@ -177,7 +177,7 @@ bool CClaimTrieCacheNormalizationFork::unnormalizeAllNamesInTrieIfNecessary() db << "UPDATE support SET nodeName = name"; // we need to let the tree structure method do the actual node delete - db << "UPDATE node SET hash = NULL WHERE name NOT IN " + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN " "(SELECT DISTINCT name FROM claim)"; return true; @@ -243,10 +243,10 @@ uint256 ComputeMerkleRoot(std::vector hashes) return hashes.empty() ? uint256{} : hashes[0]; } -uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, int takeoverHeight) +uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight) { if (nNextHeight < base->nAllClaimsInMerkleForkHeight) - return CClaimTrieCacheNormalizationFork::computeNodeHash(name, takeoverHeight); + return CClaimTrieCacheNormalizationFork::computeNodeHash(name, claimsHash, takeoverHeight); std::vector childHashes; childHashQuery << name >> [&childHashes](std::string, uint256 hash) { @@ -254,24 +254,28 @@ uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, int ta }; childHashQuery++; - std::vector claimHashes; if (takeoverHeight > 0) { - COutPoint p; - for (auto &&row: claimHashQuery << nNextHeight << name) { - row >> p.hash >> p.n; - claimHashes.push_back(getValueHash(p, takeoverHeight)); + if (claimsHash.IsNull()) { + COutPoint p; + std::vector hashes; + for (auto&& row: claimHashQuery << nNextHeight << name) { + row >> p.hash >> p.n; + hashes.push_back(getValueHash(p, takeoverHeight)); + } + claimHashQuery++; + claimsHash = hashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(hashes)); } - claimHashQuery++; + } else { + claimsHash = emptyHash; } - if (name.empty() && childHashes.empty() && claimHashes.empty() + if (name.empty() && childHashes.empty() && claimsHash == emptyHash && base->nMaxRemovalWorkaroundHeight < 0) // detecting regtest, but maybe all on next hard-fork? return emptyTrieHash; // here for compatibility with the functional tests - auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes)); - auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(claimHashes)); + const auto& childrenHash = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes)); - return Hash(left.begin(), left.end(), right.begin(), right.end()); + return Hash(childrenHash.begin(), childrenHash.end(), claimsHash.begin(), claimsHash.end()); } std::vector ComputeMerklePath(const std::vector& hashes, uint32_t idx) @@ -402,7 +406,7 @@ void CClaimTrieCacheHashFork::initializeIncrement() // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests) if (nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) { ensureTransacting(); - db << "UPDATE node SET hash = NULL"; + db << "UPDATE node SET hash = NULL, claimsHash = NULL"; } } @@ -411,7 +415,7 @@ bool CClaimTrieCacheHashFork::finalizeDecrement() auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement(); if (ret && nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) { ensureTransacting(); - db << "UPDATE node SET hash = NULL"; + db << "UPDATE node SET hash = NULL, claimsHash = NULL"; } return ret; } diff --git a/src/claimtrie/forks.h b/src/claimtrie/forks.h index f39d01811..8ef522cc3 100644 --- a/src/claimtrie/forks.h +++ b/src/claimtrie/forks.h @@ -66,7 +66,7 @@ public: bool allowSupportMetadata() const; protected: - uint256 computeNodeHash(const std::string& name, int takeoverHeight) override; + uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight) override; }; typedef CClaimTrieCacheHashFork CClaimTrieCache; diff --git a/src/claimtrie/trie.cpp b/src/claimtrie/trie.cpp index 982fb208d..86da57bda 100644 --- a/src/claimtrie/trie.cpp +++ b/src/claimtrie/trie.cpp @@ -73,7 +73,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, db << "CREATE TABLE IF NOT EXISTS node (name BLOB NOT NULL PRIMARY KEY, " "parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " - "hash BLOB)"; + "hash BLOB, claimsHash BLOB)"; db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL PRIMARY KEY, name BLOB NOT NULL, " "nodeName BLOB NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " @@ -96,6 +96,8 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, db << "DELETE FROM takeover"; } + doNodeTableMigration(); + db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)"; // db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)"; @@ -114,6 +116,29 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, db << "INSERT OR IGNORE INTO node(name, hash) VALUES(x'', ?)" << emptyTrieHash; // ensure that we always have our root node } +void CClaimTrie::doNodeTableMigration() +{ + try { + isNodeMigrationStart = false; + for (auto&& row : db << "SELECT claimsHash FROM node WHERE name = x''") + break; + } catch (const sqlite::sqlite_exception&) { + + isNodeMigrationStart = true; + + // new node schema + db << "CREATE TABLE node_new (name BLOB NOT NULL PRIMARY KEY, " + "parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " + "hash BLOB, claimsHash BLOB)"; + + db << "INSERT OR REPLACE INTO node_new(name, parent, hash) " + "SELECT name, parent, hash FROM node"; + + db << "DROP TABLE node"; + db << "ALTER TABLE node_new RENAME TO node"; + } +} + CClaimTrieCacheBase::~CClaimTrieCacheBase() { if (transacting) { @@ -279,7 +304,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() "name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " "SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix)"; - auto insertQuery = db << "INSERT INTO node(name, parent, hash) VALUES(?, ?, NULL) " + auto insertQuery = db << "INSERT INTO node(name, parent, hash, claimsHash) VALUES(?, ?, NULL, NULL) " "ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL"; auto nodeQuery = db << "SELECT name FROM node WHERE parent = ?"; @@ -438,7 +463,7 @@ void completeHash(uint256& partialHash, const std::string& key, int to) partialHash = Hash(it, it + 1, partialHash.begin(), partialHash.end()); } -uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, int takeoverHeight) +uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight) { const auto pos = name.size(); std::vector vchToHash; @@ -451,11 +476,13 @@ uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, int takeov childHashQuery++; if (takeoverHeight > 0) { - CClaimValue claim; - if (getInfoForName(name, claim)) { - auto valueHash = getValueHash(claim.outPoint, takeoverHeight); - vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end()); + if (claimsHash.IsNull()) { + CClaimValue claim; + if (getInfoForName(name, claim)) + claimsHash = getValueHash(claim.outPoint, takeoverHeight); } + if (!claimsHash.IsNull()) + vchToHash.insert(vchToHash.end(), claimsHash.begin(), claimsHash.end()); } return vchToHash.empty() ? emptyTrieHash : Hash(vchToHash.begin(), vchToHash.end()); @@ -473,22 +500,47 @@ bool CClaimTrieCacheBase::checkConsistency() } } + if (base->isNodeMigrationStart) + ensureTransacting(); + + auto updateQuery = db << "UPDATE node SET claimsHash = ? WHERE name = ?"; // not checking everything as it takes too long - auto query = db << "SELECT n.name, n.hash, " + auto query = db << (base->isNodeMigrationStart ? + "SELECT n.name, n.hash, n.claimsHash, " "IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END " - "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM node n " - "WHERE n.name IN (SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 100000) OR n.parent = x''"; + "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) " + "FROM node n ORDER BY LENGTH(n.name) DESC" + : + "SELECT n.name, n.hash, n.claimsHash, " + "IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END " + "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) " + "FROM node n WHERE n.name IN " + "(SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 100000) OR n.parent = x''"); for (auto&& row: query) { std::string name; - uint256 hash; int takeoverHeight; - row >> name >> hash >> takeoverHeight; - auto computedHash = computeNodeHash(name, takeoverHeight); + uint256 hash, claimsHash, computedClaimsHash; + row >> name >> hash >> claimsHash >> takeoverHeight; + auto computedHash = computeNodeHash(name, computedClaimsHash, takeoverHeight); if (computedHash != hash) { logPrint << "Invalid hash at " << name << Clog::endl; return false; } + if (base->isNodeMigrationStart) { + assert(!computedClaimsHash.IsNull()); + updateQuery << computedClaimsHash << name; + updateQuery++; + } else if (computedClaimsHash != claimsHash) { + logPrint << "Invalid claimsHash at " << name << Clog::endl; + return false; + } } + + updateQuery.used(true); + + if (base->isNodeMigrationStart) + return flush(); + return true; } @@ -592,20 +644,20 @@ uint256 CClaimTrieCacheBase::getMerkleHash() { ensureTreeStructureIsUpToDate(); uint256 hash; - db << "SELECT hash FROM node WHERE name = x''" - >> [&hash](std::unique_ptr rootHash) { - if (rootHash) - hash = std::move(*rootHash); - }; - if (!hash.IsNull()) - return hash; + for (auto&& row : db << "SELECT hash FROM node WHERE name = x''") { + row >> hash; + if (!hash.IsNull()) + return hash; + } assert(transacting); // no data changed but we didn't have the root hash there already? - auto updateQuery = db << "UPDATE node SET hash = ? WHERE name = ?"; - db << "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END FROM takeover t WHERE t.name = n.name " - "ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob - >> [this, &hash, &updateQuery](const std::string& name, int takeoverHeight) { - hash = computeNodeHash(name, takeoverHeight); - updateQuery << hash << name; + auto updateQuery = db << "UPDATE node SET hash = ?, claimsHash = ? WHERE name = ?"; + db << "SELECT n.name, n.claimsHash, " + "IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END " + "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) " + "FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob + >> [&](const std::string& name, uint256 claimsHash, int takeoverHeight) { + hash = computeNodeHash(name, claimsHash, takeoverHeight); + updateQuery << hash << claimsHash << name; updateQuery++; }; updateQuery.used(true); @@ -659,7 +711,7 @@ bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& out << originalHeight << nHeight << nValidHeight << nValidHeight << expires; if (nValidHeight < nNextHeight) - db << "INSERT INTO node(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; + db << "INSERT INTO node(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nodeName; return true; } @@ -680,7 +732,7 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o << supportedClaimId << name << nodeName << outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << nValidHeight << expires; if (nValidHeight < nNextHeight) - db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName; + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName; return true; } @@ -709,7 +761,7 @@ bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, const COutPoint& o if (!db.rows_modified()) return false; - db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName; + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName; // when node should be deleted from cache but instead it's kept // because it's a parent one and should not be effectively erased @@ -740,7 +792,8 @@ bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string& db << "DELETE FROM support WHERE txID = ? AND txN = ?" << outPoint.hash << outPoint.n; if (!db.rows_modified()) return false; - db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName; + + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName; return true; } @@ -757,11 +810,11 @@ bool CClaimTrieCacheBase::incrementBlock() db << "INSERT INTO node(name) SELECT nodeName FROM claim INDEXED BY claim_activationHeight " "WHERE activationHeight = ?1 AND expirationHeight > ?1 " - "ON CONFLICT(name) DO UPDATE SET hash = NULL" + "ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight; // don't make new nodes for items in supports or items that expire this block that don't exist in claims - db << "UPDATE node SET hash = NULL WHERE name IN " + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN " "(SELECT nodeName FROM claim WHERE expirationHeight = ?1 " "UNION SELECT nodeName FROM support WHERE expirationHeight = ?1 OR activationHeight = ?1)" << nNextHeight; @@ -836,10 +889,10 @@ bool CClaimTrieCacheBase::decrementBlock() nNextHeight--; db << "INSERT INTO node(name) SELECT nodeName FROM claim " - "WHERE expirationHeight = ? ON CONFLICT(name) DO UPDATE SET hash = NULL" + "WHERE expirationHeight = ? ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight; - db << "UPDATE node SET hash = NULL WHERE name IN(" + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN(" "SELECT nodeName FROM support WHERE expirationHeight = ?1 OR activationHeight = ?1 " "UNION SELECT nodeName FROM claim WHERE activationHeight = ?1)" << nNextHeight; @@ -855,7 +908,7 @@ bool CClaimTrieCacheBase::decrementBlock() bool CClaimTrieCacheBase::finalizeDecrement() { - db << "UPDATE node SET hash = NULL WHERE name IN " + db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN " "(SELECT nodeName FROM claim WHERE activationHeight = ?1 AND expirationHeight > ?1 " "UNION SELECT nodeName FROM support WHERE activationHeight = ?1 AND expirationHeight > ?1 " "UNION SELECT name FROM takeover WHERE height = ?1)" << nNextHeight; diff --git a/src/claimtrie/trie.h b/src/claimtrie/trie.h index b8e87a73f..63a0832ef 100644 --- a/src/claimtrie/trie.h +++ b/src/claimtrie/trie.h @@ -52,6 +52,7 @@ protected: const std::size_t dbCacheBytes; const std::string dbFile; sqlite::database db; + bool isNodeMigrationStart; const int nProportionalDelayFactor; const int nNormalizedNameForkHeight; @@ -61,6 +62,9 @@ protected: const int64_t nExtendedClaimExpirationTime; const int64_t nExtendedClaimExpirationForkHeight; const int64_t nAllClaimsInMerkleForkHeight; + +private: + void doNodeTableMigration(); }; class CClaimTrieCacheBase @@ -124,7 +128,7 @@ protected: mutable std::unordered_set removalWorkaround; sqlite::database_binder childHashQuery, claimHashQuery, claimHashQueryLimit; - virtual uint256 computeNodeHash(const std::string& name, int takeoverHeight); + virtual uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight); supportEntryType getSupportsForName(const std::string& name) const; virtual int getDelayForName(const std::string& name, const uint160& claimId) const; diff --git a/src/validation.cpp b/src/validation.cpp index ac39ba81a..c94437930 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2347,6 +2347,9 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // TODO: if the "just check" flag is set, we should reduce the work done here. Incrementing blocks twice per mine is not efficient. assert(trieCache.incrementBlock()); + int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; + LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); + if (trieCache.getMerkleHash() != block.hashClaimTrie) { return state.Invalid(ValidationInvalidReason::CLAIMTRIE_HASH, error("ConnectBlock() : the merkle root of the claim trie does not match " @@ -2354,9 +2357,6 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl block.hashClaimTrie.GetHex(), pindex->nHeight), REJECT_INVALID, "bad-claim-merkle-hash"); } - int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; - LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); - CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); if (block.vtx[0]->GetValueOut() > blockReward) return state.Invalid(ValidationInvalidReason::CONSENSUS, @@ -2366,8 +2366,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl if (!control.Wait()) return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); - int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; - LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); + int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime3; + LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime3), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime3) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); if (fJustCheck) return true;