LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

made another test pass, cleaned up cruft

Browse source
This commit is contained in:
Brannon King 2019-10-24 15:21:36 -06:00
parent c2d2e4befd
commit 86ae72abb2
9 changed files with 182 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

253
src/claimtrie.cpp
View file

@ -41,91 +41,94 @@ uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover)
return result; return result;
} }
CClaimTrie::CClaimTrie(bool fMemory, bool fWipe, int height, int proportionalDelayFactor, std::size_t cacheMB) static const sqlite::sqlite_config sharedConfig{
: _db(fMemory ? ":memory:" : (GetDataDir() / "claims.sqlite").string()) sqlite::OpenFlags::READWRITE | sqlite::OpenFlags::CREATE, // TODO: test with this: | sqlite::OpenFlags::SHAREDCACHE,
nullptr, sqlite::Encoding::UTF8
};
CClaimTrie::CClaimTrie(bool fWipe, int height, int proportionalDelayFactor, std::size_t cacheMB)
: dbPath((GetDataDir() / "claims.sqlite").string()), db(dbPath, sharedConfig),
nNextHeight(height), nProportionalDelayFactor(proportionalDelayFactor)
{ {
nNextHeight = height; db.define("merkle_root", [](std::vector<uint256>& hashes, const std::vector<unsigned char>& blob) { hashes.emplace_back(uint256(blob)); },
nProportionalDelayFactor = proportionalDelayFactor; [](const std::vector<uint256>& hashes) { return ComputeMerkleRoot(hashes); });
_db.define("merkle_root", [](std::vector<uint256>& hashes, const std::vector<unsigned char>& blob) { hashes.emplace_back(uint256(blob)); }, db.define("merkle_pair", [](const std::vector<unsigned char>& blob1, const std::vector<unsigned char>& blob2) { return Hash(blob1.begin(), blob1.end(), blob2.begin(), blob2.end()); });
[](const std::vector<uint256>& hashes) { return ComputeMerkleRoot(hashes); }); db.define("merkle", [](const std::vector<unsigned char>& blob1) { return Hash(blob1.begin(), blob1.end()); });
_db.define("merkle_pair", [](const std::vector<unsigned char>& blob1, const std::vector<unsigned char>& blob2) { return Hash(blob1.begin(), blob1.end(), blob2.begin(), blob2.end()); }); db << "CREATE TABLE IF NOT EXISTS nodes (name TEXT NOT NULL PRIMARY KEY, parent TEXT, hash BLOB)";
_db.define("merkle", [](const std::vector<unsigned char>& blob1) { return Hash(blob1.begin(), blob1.end()); }); db << "CREATE INDEX IF NOT EXISTS nodes_hash ON nodes (hash)";
db << "CREATE INDEX IF NOT EXISTS nodes_parent ON nodes (parent)";
_db << "CREATE TABLE IF NOT EXISTS nodes (name TEXT NOT NULL PRIMARY KEY, parent TEXT, hash BLOB)"; db << "CREATE TABLE IF NOT EXISTS claims (claimID BLOB NOT NULL PRIMARY KEY, name TEXT NOT NULL, "
_db << "CREATE INDEX nodes_hash ON nodes (hash)";
_db << "CREATE INDEX nodes_parent ON nodes (parent)";
_db << "CREATE TABLE IF NOT EXISTS claims (claimID BLOB NOT NULL PRIMARY KEY, name TEXT NOT NULL, "
"nodeName TEXT NOT NULL REFERENCES nodes(name) DEFERRABLE INITIALLY DEFERRED, " "nodeName TEXT NOT NULL REFERENCES nodes(name) DEFERRABLE INITIALLY DEFERRED, "
"txID BLOB NOT NULL, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, " "txID BLOB NOT NULL, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, "
"validHeight INTEGER NOT NULL, expirationHeight INTEGER NOT NULL, " "validHeight INTEGER NOT NULL, expirationHeight INTEGER NOT NULL, "
"amount INTEGER NOT NULL, metadata BLOB);"; "amount INTEGER NOT NULL, metadata BLOB);";
_db << "CREATE INDEX claims_validHeight ON claims (validHeight)"; db << "CREATE INDEX IF NOT EXISTS claims_validHeight ON claims (validHeight)";
_db << "CREATE INDEX claims_expirationHeight ON claims (expirationHeight)"; db << "CREATE INDEX IF NOT EXISTS claims_expirationHeight ON claims (expirationHeight)";
_db << "CREATE INDEX claims_nodeName ON claims (nodeName)"; db << "CREATE INDEX IF NOT EXISTS claims_nodeName ON claims (nodeName)";
_db << "CREATE TABLE IF NOT EXISTS supports (txID BLOB NOT NULL, txN INTEGER NOT NULL, " db << "CREATE TABLE IF NOT EXISTS supports (txID BLOB NOT NULL, txN INTEGER NOT NULL, "
"supportedClaimID BLOB NOT NULL, name TEXT NOT NULL, nodeName TEXT NOT NULL, " "supportedClaimID BLOB NOT NULL, name TEXT NOT NULL, nodeName TEXT NOT NULL, "
"blockHeight INTEGER NOT NULL, validHeight INTEGER NOT NULL, expirationHeight INTEGER NOT NULL, " "blockHeight INTEGER NOT NULL, validHeight INTEGER NOT NULL, expirationHeight INTEGER NOT NULL, "
"amount INTEGER NOT NULL, metadata BLOB, PRIMARY KEY(txID, txN));"; "amount INTEGER NOT NULL, metadata BLOB, PRIMARY KEY(txID, txN));";
_db << "CREATE INDEX supports_supportedClaimID ON supports (supportedClaimID)"; db << "CREATE INDEX IF NOT EXISTS supports_supportedClaimID ON supports (supportedClaimID)";
_db << "CREATE INDEX supports_validHeight ON supports (validHeight)"; db << "CREATE INDEX IF NOT EXISTS supports_validHeight ON supports (validHeight)";
_db << "CREATE INDEX supports_expirationHeight ON supports (expirationHeight)"; db << "CREATE INDEX IF NOT EXISTS supports_expirationHeight ON supports (expirationHeight)";
_db << "CREATE INDEX supports_nodeName ON supports (nodeName)"; db << "CREATE INDEX IF NOT EXISTS supports_nodeName ON supports (nodeName)";
_db << "PRAGMA cache_size=-" + std::to_string(cacheMB * 1024); // in -KB db << "PRAGMA cache_size=-" + std::to_string(cacheMB * 1024); // in -KB
_db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit
_db << "PRAGMA journal_mode=WAL"; // PRAGMA wal_autocheckpoint=10000; db << "PRAGMA journal_mode=WAL"; // PRAGMA wal_autocheckpoint=10000;
_db << "PRAGMA case_sensitive_like=true"; db << "PRAGMA case_sensitive_like=true";
if (fWipe) { if (fWipe) {
_db << "DELETE FROM nodes"; db << "DELETE FROM nodes";
_db << "DELETE FROM claims"; db << "DELETE FROM claims";
_db << "DELETE FROM supports"; db << "DELETE FROM supports";
} }
_db << "INSERT OR IGNORE INTO nodes(name, hash) VALUES('', ?)" << uint256(); // ensure that we always have our root node db << "INSERT OR IGNORE INTO nodes(name, hash) VALUES('', ?)" << one; // ensure that we always have our root node
} }
CClaimTrieCacheBase::~CClaimTrieCacheBase() { CClaimTrieCacheBase::~CClaimTrieCacheBase() {
base->_db << "rollback"; db << "rollback";
} }
bool CClaimTrie::SyncToDisk() bool CClaimTrie::SyncToDisk()
{ {
// alternatively, switch to full sync after we are caught up on the chain // alternatively, switch to full sync after we are caught up on the chain
auto rc = sqlite3_wal_checkpoint_v2(_db.connection().get(), nullptr, SQLITE_CHECKPOINT_FULL, nullptr, nullptr); auto rc = sqlite3_wal_checkpoint_v2(db.connection().get(), nullptr, SQLITE_CHECKPOINT_FULL, nullptr, nullptr);
return rc == SQLITE_OK; return rc == SQLITE_OK;
} }
bool CClaimTrie::empty() { bool CClaimTrie::empty() {
int64_t count; int64_t count;
_db << "SELECT COUNT(*) FROM claims" >> count; db << "SELECT COUNT(*) FROM claims" >> count;
return count == 0; return count == 0;
} }
bool CClaimTrieCacheBase::haveClaim(const std::string& name, const COutPoint& outPoint) const bool CClaimTrieCacheBase::haveClaim(const std::string& name, const COutPoint& outPoint) const
{ {
auto query = base->_db << "SELECT 1 FROM claims WHERE nodeName = ? AND txID = ? AND txN = ? " auto query = db << "SELECT 1 FROM claims WHERE nodeName = ? AND txID = ? AND txN = ? "
"AND validHeight < ? AND expirationHeight >= ? LIMIT 1" "AND validHeight < ? AND expirationHeight >= ? LIMIT 1"
<< name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight; << name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight;
return query.begin() != query.end(); return query.begin() != query.end();
} }
bool CClaimTrieCacheBase::haveSupport(const std::string& name, const COutPoint& outPoint) const bool CClaimTrieCacheBase::haveSupport(const std::string& name, const COutPoint& outPoint) const
{ {
auto query = base->_db << "SELECT 1 FROM supports WHERE nodeName = ? AND txID = ? AND txN = ? " auto query = db << "SELECT 1 FROM supports WHERE nodeName = ? AND txID = ? AND txN = ? "
"AND validHeight < ? AND expirationHeight >= ? LIMIT 1" "AND validHeight < ? AND expirationHeight >= ? LIMIT 1"
<< name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight; << name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight;
return query.begin() != query.end(); return query.begin() != query.end();
} }
supportEntryType CClaimTrieCacheBase::getSupportsForName(const std::string& name) const supportEntryType CClaimTrieCacheBase::getSupportsForName(const std::string& name) const
{ {
// includes values that are not yet valid // includes values that are not yet valid
auto query = base->_db << "SELECT supportedClaimID, txID, txN, blockHeight, validHeight, amount " auto query = db << "SELECT supportedClaimID, txID, txN, blockHeight, validHeight, amount "
"FROM supports WHERE nodeName = ? AND expirationHeight >= ?" << name << nNextHeight; "FROM supports WHERE nodeName = ? AND expirationHeight >= ?" << name << nNextHeight;
supportEntryType ret; supportEntryType ret;
for (auto&& row: query) { for (auto&& row: query) {
@ -139,9 +142,9 @@ supportEntryType CClaimTrieCacheBase::getSupportsForName(const std::string& name
bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const
{ {
auto query = base->_db << "SELECT validHeight FROM claims WHERE nodeName = ? AND txID = ? AND txN = ? " auto query = db << "SELECT validHeight FROM claims WHERE nodeName = ? AND txID = ? AND txN = ? "
"AND validHeight >= ? AND expirationHeight >= ? LIMIT 1" "AND validHeight >= ? AND expirationHeight >= ? LIMIT 1"
<< name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight; << name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight;
for (auto&& row: query) { for (auto&& row: query) {
row >> nValidAtHeight; row >> nValidAtHeight;
return true; return true;
@ -151,9 +154,9 @@ bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPo
bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const
{ {
auto query = base->_db << "SELECT validHeight FROM supports WHERE nodeName = ? AND txID = ? AND txN = ? " auto query = db << "SELECT validHeight FROM supports WHERE nodeName = ? AND txID = ? AND txN = ? "
"AND validHeight >= ? AND expirationHeight >= ? LIMIT 1" "AND validHeight >= ? AND expirationHeight >= ? LIMIT 1"
<< name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight; << name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight;
for (auto&& row: query) { for (auto&& row: query) {
row >> nValidAtHeight; row >> nValidAtHeight;
return true; return true;
@ -174,49 +177,48 @@ bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::str
if (name.empty()) return false; if (name.empty()) return false;
// to remove a node it must have one or less children and no claims // to remove a node it must have one or less children and no claims
vector_builder<std::string, std::string> claimsBuilder; vector_builder<std::string, std::string> claimsBuilder;
base->_db << "SELECT name FROM claims WHERE name = ? AND validHeight < ? AND expirationHeight >= ? " db << "SELECT name FROM claims WHERE name = ? AND validHeight < ? AND expirationHeight >= ? "
<< name << nNextHeight << nNextHeight >> claimsBuilder; << name << nNextHeight << nNextHeight >> claimsBuilder;
claims = std::move(claimsBuilder); claims = std::move(claimsBuilder);
if (!claims.empty()) return false; // still has claims if (!claims.empty()) return false; // still has claims
// we now know it has no claims, but we need to check its children // we now know it has no claims, but we need to check its children
int64_t count; int64_t count;
std::string childName; std::string childName;
base->_db << "SELECT COUNT(*),MAX(name) FROM nodes WHERE parent = ?" << name >> std::tie(count, childName); db << "SELECT COUNT(*),MAX(name) FROM nodes WHERE parent = ?" << name >> std::tie(count, childName);
if (count > 1) return false; // still has multiple children if (count > 1) return false; // still has multiple children
// okay. it's going away // okay. it's going away
auto query = base->_db << "SELECT parent FROM nodes WHERE name = ?" << name; auto query = db << "SELECT parent FROM nodes WHERE name = ?" << name;
auto it = query.begin(); auto it = query.begin();
if (it == query.end()) if (it == query.end())
return true; // we'll assume that whoever deleted this node previously cleaned things up correctly return true; // we'll assume that whoever deleted this node previously cleaned things up correctly
*it >> parent; *it >> parent;
base->_db << "DELETE FROM nodes WHERE name = ?" << name; db << "DELETE FROM nodes WHERE name = ?" << name;
auto ret = base->_db.rows_modified() > 0; auto ret = db.rows_modified() > 0;
if (ret && count == 1) // make the child skip us and point to its grandparent: if (ret && count == 1) // make the child skip us and point to its grandparent:
base->_db << "UPDATE nodes SET parent = ? WHERE name = ?" << parent << childName; db << "UPDATE nodes SET parent = ? WHERE name = ?" << parent << childName;
if (ret) if (ret)
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent;
return ret; return ret;
} }
void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
if (!dirtyNodes) return;
dirtyNodes = false;
// your children are your nodes that match your key, go at least one longer, // your children are your nodes that match your key, go at least one longer,
// and have nothing in common with the other nodes in that set -- a hard query w/o parent // and have nothing in common with the other nodes in that set -- a hard query w/o parent
// the plan: update all the claim hashes first // the plan: update all the claim hashes first
vector_builder<std::string, std::string> names; vector_builder<std::string, std::string> names;
base->_db << "SELECT name FROM nodes WHERE hash IS NULL ORDER BY LENGTH(name) DESC, name DESC" >> names; db << "SELECT name FROM nodes WHERE hash IS NULL ORDER BY LENGTH(name) DESC, name DESC" >> names;
if (names.empty()) return; // could track this with a dirty flag to avoid the above query (but it would be some maintenance)
// there's an assumption that all nodes with claims are here; we do that as claims are inserted // there's an assumption that all nodes with claims are here; we do that as claims are inserted
// should we do the same to remove nodes? no; we need their last takeover height if they come back // should we do the same to remove nodes? no; we need their last takeover height if they come back
//float time = 0; //float time = 0;
// assume parents are not set correctly here: // assume parents are not set correctly here:
auto parentQuery = base->_db << "SELECT name FROM nodes WHERE parent IS NOT NULL AND " auto parentQuery = db << "SELECT name FROM nodes WHERE parent IS NOT NULL AND "
"name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " "name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
"SELECT SUBSTR(p, 1, LENGTH(p)) FROM prefix WHERE p != '') SELECT p FROM prefix) " "SELECT SUBSTR(p, 1, LENGTH(p)-1) FROM prefix WHERE p != '') SELECT p FROM prefix) "
"ORDER BY LENGTH(name) DESC LIMIT 1"; "ORDER BY LENGTH(name) DESC LIMIT 1";
for (auto& name: names) { for (auto& name: names) {
@ -240,7 +242,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
// if it already exists: // if it already exists:
if (nameOrParent == name) { // TODO: we could actually set the hash here if (nameOrParent == name) { // TODO: we could actually set the hash here
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << name; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << name;
continue; continue;
} }
@ -249,7 +251,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
// we know now that we need to insert it, // we know now that we need to insert it,
// but we may need to insert a parent node for it first (also called a split) // but we may need to insert a parent node for it first (also called a split)
vector_builder<std::string, std::string> siblings; vector_builder<std::string, std::string> siblings;
base->_db << "SELECT name FROM nodes WHERE parent = ?" << parent >> siblings; db << "SELECT name FROM nodes WHERE parent = ?" << parent >> siblings;
std::size_t splitPos = 0; std::size_t splitPos = 0;
for (auto& sibling: siblings) { for (auto& sibling: siblings) {
if (sibling[parent.size()] == name[parent.size()]) { if (sibling[parent.size()] == name[parent.size()]) {
@ -258,15 +260,15 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
++splitPos; ++splitPos;
auto newNodeName = name.substr(0, splitPos); auto newNodeName = name.substr(0, splitPos);
// notify new node's parent: // notify new node's parent:
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent;
// and his sibling: // and his sibling:
base->_db << "UPDATE nodes SET parent = ? WHERE name = ?" << newNodeName << sibling; db << "UPDATE nodes SET parent = ? WHERE name = ?" << newNodeName << sibling;
if (splitPos == name.size()) { if (splitPos == name.size()) {
// our new node is the same as the one we wanted to insert // our new node is the same as the one we wanted to insert
break; break;
} }
// insert the split node: // insert the split node:
base->_db << "INSERT INTO nodes(name, parent, hash) VALUES(?, ?, NULL) " db << "INSERT INTO nodes(name, parent, hash) VALUES(?, ?, NULL) "
"ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL" "ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL"
<< newNodeName << parent; << newNodeName << parent;
parent = std::move(newNodeName); parent = std::move(newNodeName);
@ -274,25 +276,25 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
} }
} }
base->_db << "INSERT INTO nodes(name, parent, hash) VALUES(?, ?, NULL) " db << "INSERT INTO nodes(name, parent, hash) VALUES(?, ?, NULL) "
"ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL" "ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL"
<< name << parent; << name << parent;
if (splitPos == 0) if (splitPos == 0)
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << parent;
} }
// now we need to percolate the nulls up the tree // now we need to percolate the nulls up the tree
// parents should all be set right // parents should all be set right
base->_db << "UPDATE nodes SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS (" db << "UPDATE nodes SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS "
"SELECT parent WHERE hash IS NULL ORDER BY name DESC UNION " "(SELECT parent FROM nodes WHERE hash IS NULL UNION SELECT parent FROM prefix, nodes "
"SELECT parent FROM prefix,nodes WHERE name = p AND p IS NOT NULL)"; "WHERE name = prefix.p AND prefix.p IS NOT NULL ORDER BY parent DESC) SELECT p FROM prefix)";
} }
std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const
{ {
// you could do this select from the nodes table, but you would have to ensure it is not dirty first // you could do this select from the nodes table, but you would have to ensure it is not dirty first
std::size_t ret; std::size_t ret;
base->_db << "SELECT COUNT(DISTINCT nodeName) FROM claims WHERE validHeight < ? AND expirationHeight >= ?" db << "SELECT COUNT(DISTINCT nodeName) FROM claims WHERE validHeight < ? AND expirationHeight >= ?"
<< nNextHeight << nNextHeight >> ret; << nNextHeight << nNextHeight >> ret;
return ret; return ret;
} }
@ -300,7 +302,7 @@ std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const
std::size_t CClaimTrieCacheBase::getTotalClaimsInTrie() const std::size_t CClaimTrieCacheBase::getTotalClaimsInTrie() const
{ {
std::size_t ret; std::size_t ret;
base->_db << "SELECT COUNT(*) FROM claims WHERE validHeight < ? AND expirationHeight >= ?" db << "SELECT COUNT(*) FROM claims WHERE validHeight < ? AND expirationHeight >= ?"
<< nNextHeight << nNextHeight >> ret; << nNextHeight << nNextHeight >> ret;
return ret; return ret;
} }
@ -313,17 +315,17 @@ CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly)
"FROM claims c WHERE c.validHeight < ? AND s.expirationHeight >= ?"); "FROM claims c WHERE c.validHeight < ? AND s.expirationHeight >= ?");
if (fControllingOnly) if (fControllingOnly)
throw std::runtime_error("not implemented yet"); // TODO: finish this throw std::runtime_error("not implemented yet"); // TODO: finish this
base->_db << query << nNextHeight << nNextHeight << nNextHeight << nNextHeight >> ret; db << query << nNextHeight << nNextHeight << nNextHeight << nNextHeight >> ret;
return ret; return ret;
} }
bool CClaimTrieCacheBase::getInfoForName(const std::string& name, CClaimValue& claim) const bool CClaimTrieCacheBase::getInfoForName(const std::string& name, CClaimValue& claim) const
{ {
auto query = base->_db << "SELECT c.claimID, c.txID, c.txN, c.blockHeight, c.validHeight, c.amount, " auto query = db << "SELECT c.claimID, c.txID, c.txN, c.blockHeight, c.validHeight, c.amount, "
"(SELECT TOTAL(s.amount)+c.amount FROM supports s WHERE s.supportedClaimID = c.claimID AND s.validHeight < ? AND s.expirationHeight >= ?) as effectiveAmount " "(SELECT TOTAL(s.amount)+c.amount FROM supports s WHERE s.supportedClaimID = c.claimID AND s.validHeight < ? AND s.expirationHeight >= ?) as effectiveAmount "
"FROM claims c WHERE c.nodeName = ? AND c.validHeight < ? AND c.expirationHeight >= ? " "FROM claims c WHERE c.nodeName = ? AND c.validHeight < ? AND c.expirationHeight >= ? "
"ORDER BY effectiveAmount DESC, c.blockHeight, c.txID, c.txN LIMIT 1" "ORDER BY effectiveAmount DESC, c.blockHeight, c.txID, c.txN LIMIT 1"
<< nNextHeight << nNextHeight << name << nNextHeight << nNextHeight; << nNextHeight << nNextHeight << name << nNextHeight << nNextHeight;
for (auto&& row: query) { for (auto&& row: query) {
row >> claim.claimId >> claim.outPoint.hash >> claim.outPoint.n row >> claim.claimId >> claim.outPoint.hash >> claim.outPoint.n
>> claim.nHeight >> claim.nValidAtHeight >> claim.nAmount >> claim.nEffectiveAmount; >> claim.nHeight >> claim.nValidAtHeight >> claim.nAmount >> claim.nEffectiveAmount;
@ -338,9 +340,9 @@ CClaimSupportToName CClaimTrieCacheBase::getClaimsForName(const std::string& nam
int nLastTakeoverHeight = 0; int nLastTakeoverHeight = 0;
auto supports = getSupportsForName(name); auto supports = getSupportsForName(name);
auto query = base->_db << "SELECT claimID, txID, txN, blockHeight, validHeight, amount " auto query = db << "SELECT claimID, txID, txN, blockHeight, validHeight, amount "
"FROM claims WHERE nodeName = ? AND expirationHeight >= ?" "FROM claims WHERE nodeName = ? AND expirationHeight >= ?"
<< name << nNextHeight; << name << nNextHeight;
for (auto&& row: query) { for (auto&& row: query) {
CClaimValue claim; CClaimValue claim;
row >> claim.claimId >> claim.outPoint.hash >> claim.outPoint.n row >> claim.claimId >> claim.outPoint.hash >> claim.outPoint.n
@ -387,7 +389,7 @@ uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(const std::string& name,
{ {
std::vector<uint8_t> vchToHash; std::vector<uint8_t> vchToHash;
const auto pos = name.size(); const auto pos = name.size();
auto query = base->_db << "SELECT name, hash FROM nodes WHERE parent = ? ORDER BY name" << name; auto query = db << "SELECT name, hash FROM nodes WHERE parent = ? ORDER BY name" << name;
for (auto&& row : query) { for (auto&& row : query) {
std::string key; std::string key;
std::unique_ptr<uint256> hash; std::unique_ptr<uint256> hash;
@ -409,7 +411,7 @@ uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(const std::string& name,
auto computedHash = vchToHash.empty() ? one : Hash(vchToHash.begin(), vchToHash.end()); auto computedHash = vchToHash.empty() ? one : Hash(vchToHash.begin(), vchToHash.end());
if (!checkOnly) if (!checkOnly)
base->_db << "UPDATE nodes SET hash = ? WHERE name = ?" << computedHash << name; db << "UPDATE nodes SET hash = ? WHERE name = ?" << computedHash << name;
return computedHash; return computedHash;
} }
@ -417,7 +419,7 @@ bool CClaimTrieCacheBase::checkConsistency()
{ {
// verify that all claims hash to the values on the nodes // verify that all claims hash to the values on the nodes
auto query = base->_db << "SELECT name, hash FROM nodes"; auto query = db << "SELECT name, hash FROM nodes";
for (auto&& row: query) { for (auto&& row: query) {
std::string name; std::string name;
uint256 hash; uint256 hash;
@ -433,8 +435,8 @@ bool CClaimTrieCacheBase::flush()
{ {
getMerkleHash(); getMerkleHash();
try { try {
base->_db << "commit"; db << "commit";
base->_db << "begin"; db << "begin";
} }
catch (const std::exception& e) { catch (const std::exception& e) {
LogPrintf("ERROR in ClaimTrieCache flush: %s\n", e.what()); LogPrintf("ERROR in ClaimTrieCache flush: %s\n", e.what());
@ -456,7 +458,7 @@ bool CClaimTrieCacheBase::ValidateTipMatches(const CBlockIndex* tip)
// eh, we better blow it away; it's their job to back up the folder first // eh, we better blow it away; it's their job to back up the folder first
// well, only do it if we're empty on the sqlite side -- aka, we haven't trie to sync first // well, only do it if we're empty on the sqlite side -- aka, we haven't trie to sync first
std::size_t count; std::size_t count;
base->_db << "SELECT COUNT(*) FROM nodes" >> count; db << "SELECT COUNT(*) FROM nodes" >> count;
if (!count) { if (!count) {
auto oldDataPath = GetDataDir() / "claimtrie"; auto oldDataPath = GetDataDir() / "claimtrie";
boost::system::error_code ec; boost::system::error_code ec;
@ -472,11 +474,12 @@ bool CClaimTrieCacheBase::ValidateTipMatches(const CBlockIndex* tip)
return false; return false;
} }
CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) : base(base), dirtyNodes(false) CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
: base(base), db(base->dbPath, sharedConfig)
{ {
assert(base); assert(base);
nNextHeight = base->nNextHeight; nNextHeight = base->nNextHeight;
base->_db << "begin"; db << "begin";
} }
int CClaimTrieCacheBase::expirationTime() const int CClaimTrieCacheBase::expirationTime() const
@ -488,7 +491,7 @@ uint256 CClaimTrieCacheBase::getMerkleHash()
{ {
ensureTreeStructureIsUpToDate(); ensureTreeStructureIsUpToDate();
std::unique_ptr<uint256> hash; std::unique_ptr<uint256> hash;
base->_db << "SELECT hash FROM nodes WHERE name = ''" >> hash; db << "SELECT hash FROM nodes WHERE name = ''" >> hash;
if (hash == nullptr || hash->IsNull()) if (hash == nullptr || hash->IsNull())
return recursiveComputeMerkleHash("", false); return recursiveComputeMerkleHash("", false);
return *hash; return *hash;
@ -511,13 +514,13 @@ bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& out
auto nodeName = adjustNameForValidHeight(name, nHeight + delay); auto nodeName = adjustNameForValidHeight(name, nHeight + delay);
auto expires = expirationTime() + nHeight; auto expires = expirationTime() + nHeight;
auto validHeight = nHeight + delay; auto validHeight = nHeight + delay;
base->_db << "INSERT INTO claims(claimID, name, nodeName, txID, txN, amount, blockHeight, validHeight, expirationHeight, metadata) " db << "INSERT INTO claims(claimID, name, nodeName, txID, txN, amount, blockHeight, validHeight, expirationHeight, metadata) "
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(claimID) DO UPDATE SET name = excluded.name, " "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(claimID) DO UPDATE SET name = excluded.name, "
"nodeName = excluded.nodeName, txID = excluded.txID, txN = excluded.txN, amount = excluded.amount, " "nodeName = excluded.nodeName, txID = excluded.txID, txN = excluded.txN, amount = excluded.amount, "
"expirationHeight = excluded.expirationHeight, metadata = excluded.metadata" << claimId << name << nodeName "expirationHeight = excluded.expirationHeight, metadata = excluded.metadata" << claimId << name << nodeName
<< outPoint.hash << outPoint.n << nAmount << nHeight << validHeight << expires << metadata; << outPoint.hash << outPoint.n << nAmount << nHeight << validHeight << expires << metadata;
base->_db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName;
return true; return true;
} }
@ -528,37 +531,35 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o
auto nodeName = adjustNameForValidHeight(name, nHeight + delay); auto nodeName = adjustNameForValidHeight(name, nHeight + delay);
auto expires = expirationTime() + nHeight; auto expires = expirationTime() + nHeight;
auto validHeight = nHeight + delay; auto validHeight = nHeight + delay;
base->_db << "INSERT INTO supports(supportedClaimID, name, nodeName, txID, txN, amount, blockHeight, validHeight, expirationHeight, metadata) " db << "INSERT INTO supports(supportedClaimID, name, nodeName, txID, txN, amount, blockHeight, validHeight, expirationHeight, metadata) "
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << supportedClaimId << name << nodeName "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" << supportedClaimId << name << nodeName
<< outPoint.hash << outPoint.n << nAmount << nHeight << validHeight << expires << metadata; << outPoint.hash << outPoint.n << nAmount << nHeight << validHeight << expires << metadata;
base->_db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName; db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName;
return true; return true;
} }
bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, std::string& nodeName, int& validHeight) bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, std::string& nodeName, int& validHeight)
{ {
auto query = base->_db << "SELECT nodeName, validHeight FROM claims WHERE claimID = ?" auto query = db << "SELECT nodeName, validHeight FROM claims WHERE claimID = ?"
<< claimId; << claimId;
auto it = query.begin(); auto it = query.begin();
if (it == query.end()) return false; if (it == query.end()) return false;
*it >> nodeName >> validHeight; *it >> nodeName >> validHeight;
base->_db << "DELETE FROM claims WHERE claimID = ?" << claimId; db << "DELETE FROM claims WHERE claimID = ?" << claimId;
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << nodeName; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << nodeName;
dirtyNodes = base->_db.rows_modified() > 0;
return true; return true;
} }
bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight) bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight)
{ {
auto query = base->_db << "SELECT nodeName, validHeight FROM supports WHERE txID = ? AND txN = ?" auto query = db << "SELECT nodeName, validHeight FROM supports WHERE txID = ? AND txN = ?"
<< outPoint.hash << outPoint.n; << outPoint.hash << outPoint.n;
auto it = query.begin(); auto it = query.begin();
if (it == query.end()) return false; if (it == query.end()) return false;
*it >> nodeName >> validHeight; *it >> nodeName >> validHeight;
base->_db << "DELETE FROM supports WHERE txID = ? AND txN = ?" << outPoint.hash << outPoint.n; db << "DELETE FROM supports WHERE txID = ? AND txN = ?" << outPoint.hash << outPoint.n;
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << nodeName; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << nodeName;
dirtyNodes = base->_db.rows_modified() > 0;
return true; return true;
} }
@ -858,15 +859,15 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimQueueR
// for all dirty nodes look for new takeovers // for all dirty nodes look for new takeovers
{ {
base->_db << "UPDATE nodes SET hash = NULL WHERE name IN " db << "UPDATE nodes SET hash = NULL WHERE name IN "
"(SELECT nodeName FROM claims WHERE validHeight = ?)" << nNextHeight; "(SELECT nodeName FROM claims WHERE validHeight = ?)" << nNextHeight;
base->_db << "UPDATE nodes SET hash = NULL WHERE name IN " db << "UPDATE nodes SET hash = NULL WHERE name IN "
"(SELECT nodeName FROM supports WHERE validHeight = ?)" << nNextHeight; "(SELECT nodeName FROM supports WHERE validHeight = ?)" << nNextHeight;
} }
assert(expireUndo.empty()); assert(expireUndo.empty());
{ {
auto becomingExpired = base->_db << "SELECT txID, txN, nodeName, claimID, validHeight, blockHeight, amount " auto becomingExpired = db << "SELECT txID, txN, nodeName, claimID, validHeight, blockHeight, amount "
"FROM claims WHERE expirationHeight = ?" << nNextHeight; "FROM claims WHERE expirationHeight = ?" << nNextHeight;
for (auto &&row: becomingExpired) { for (auto &&row: becomingExpired) {
CClaimValue value; CClaimValue value;
@ -876,12 +877,12 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimQueueR
expireUndo.emplace_back(name, value); expireUndo.emplace_back(name, value);
} }
} }
base->_db << "UPDATE nodes SET hash = NULL WHERE name IN (SELECT nodeName FROM claims WHERE expirationHeight = ?)" db << "UPDATE nodes SET hash = NULL WHERE name IN (SELECT nodeName FROM claims WHERE expirationHeight = ?)"
<< nNextHeight; << nNextHeight;
assert(expireSupportUndo.empty()); assert(expireSupportUndo.empty());
{ {
auto becomingExpired = base->_db << "SELECT txID, txN, nodeName, supportedClaimID, validHeight, blockHeight, amount " auto becomingExpired = db << "SELECT txID, txN, nodeName, supportedClaimID, validHeight, blockHeight, amount "
"FROM supports WHERE expirationHeight = ?" << nNextHeight; "FROM supports WHERE expirationHeight = ?" << nNextHeight;
for (auto &&row: becomingExpired) { for (auto &&row: becomingExpired) {
CSupportValue value; CSupportValue value;
@ -891,12 +892,12 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimQueueR
expireSupportUndo.emplace_back(name, value); expireSupportUndo.emplace_back(name, value);
} }
} }
base->_db << "UPDATE nodes SET hash = NULL WHERE name IN (SELECT nodeName FROM supports WHERE expirationHeight = ?)" db << "UPDATE nodes SET hash = NULL WHERE name IN (SELECT nodeName FROM supports WHERE expirationHeight = ?)"
<< nNextHeight; << nNextHeight;
// takeover handling: // takeover handling:
vector_builder<std::string, std::string> takeovers; vector_builder<std::string, std::string> takeovers;
base->_db << "SELECT name FROM nodes WHERE hash IS NULL" >> takeovers; db << "SELECT name FROM nodes WHERE hash IS NULL" >> takeovers;
for (const auto& nameWithTakeover : takeovers) { for (const auto& nameWithTakeover : takeovers) {
if (nNextHeight >= 496856 && nNextHeight <= 653524) { if (nNextHeight >= 496856 && nNextHeight <= 653524) {
@ -921,8 +922,8 @@ void CClaimTrieCacheBase::activateAllFor(insertUndoType& insertUndo, insertUndoT
const std::string& name) { const std::string& name) {
// now that we know a takeover is happening, we bring everybody in: // now that we know a takeover is happening, we bring everybody in:
{ {
auto query = base->_db << "SELECT txID, txN, validHeight FROM claims WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?" auto query = db << "SELECT txID, txN, validHeight FROM claims WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?"
<< name << nNextHeight << nNextHeight; << name << nNextHeight << nNextHeight;
for (auto &&row: query) { for (auto &&row: query) {
uint256 hash; uint256 hash;
uint32_t n; uint32_t n;
@ -932,13 +933,13 @@ void CClaimTrieCacheBase::activateAllFor(insertUndoType& insertUndo, insertUndoT
} }
} }
// and then update them all to activate now: // and then update them all to activate now:
base->_db << "UPDATE claims SET validHeight = ? WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?" db << "UPDATE claims SET validHeight = ? WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?"
<< nNextHeight << name << nNextHeight << nNextHeight; << nNextHeight << name << nNextHeight << nNextHeight;
// then do the same for supports: // then do the same for supports:
{ {
auto query = base->_db << "SELECT txID, txN, validHeight FROM supports WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?" auto query = db << "SELECT txID, txN, validHeight FROM supports WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?"
<< name << nNextHeight << nNextHeight; << name << nNextHeight << nNextHeight;
for (auto &&row: query) { for (auto &&row: query) {
uint256 hash; uint256 hash;
uint32_t n; uint32_t n;
@ -948,7 +949,7 @@ void CClaimTrieCacheBase::activateAllFor(insertUndoType& insertUndo, insertUndoT
} }
} }
// and then update them all to activate now: // and then update them all to activate now:
base->_db << "UPDATE supports SET validHeight = ? WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?" db << "UPDATE supports SET validHeight = ? WHERE nodeName = ? AND validHeight > ? AND expirationHeight >= ?"
<< nNextHeight << name << nNextHeight << nNextHeight; << nNextHeight << name << nNextHeight << nNextHeight;
} }
@ -957,27 +958,27 @@ bool CClaimTrieCacheBase::decrementBlock(insertUndoType& insertUndo, claimQueueR
nNextHeight--; nNextHeight--;
for (auto it = expireSupportUndo.crbegin(); it != expireSupportUndo.crend(); ++it) { for (auto it = expireSupportUndo.crbegin(); it != expireSupportUndo.crend(); ++it) {
base->_db << "UPDATE supports SET validHeight = ?, active = ? WHERE txID = ? AND txN = ?" db << "UPDATE supports SET validHeight = ?, active = ? WHERE txID = ? AND txN = ?"
<< it->second.nValidAtHeight << (it->second.nValidAtHeight <= nNextHeight) << it->second.outPoint.hash << it->second.outPoint.n; << it->second.nValidAtHeight << (it->second.nValidAtHeight <= nNextHeight) << it->second.outPoint.hash << it->second.outPoint.n;
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->first; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->first;
} }
for (auto it = expireUndo.crbegin(); it != expireUndo.crend(); ++it) { for (auto it = expireUndo.crbegin(); it != expireUndo.crend(); ++it) {
base->_db << "UPDATE claims SET validHeight = ?, active = ? WHERE claimID = ?" db << "UPDATE claims SET validHeight = ?, active = ? WHERE claimID = ?"
<< it->second.nValidAtHeight << (it->second.nValidAtHeight <= nNextHeight) << it->second.claimId; << it->second.nValidAtHeight << (it->second.nValidAtHeight <= nNextHeight) << it->second.claimId;
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->first; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->first;
} }
for (auto it = insertSupportUndo.crbegin(); it != insertSupportUndo.crend(); ++it) { for (auto it = insertSupportUndo.crbegin(); it != insertSupportUndo.crend(); ++it) {
base->_db << "UPDATE supports SET validHeight = ? WHERE txID = ? AND txN = ?" db << "UPDATE supports SET validHeight = ? WHERE txID = ? AND txN = ?"
<< it->outPoint.hash << it->outPoint.n; << it->outPoint.hash << it->outPoint.n;
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->name; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->name;
} }
for (auto it = insertUndo.crbegin(); it != insertUndo.crend(); ++it) { for (auto it = insertUndo.crbegin(); it != insertUndo.crend(); ++it) {
base->_db << "UPDATE claims SET validHeight = ? WHERE nodeName = ? AND txID = ? AND txN = ?" db << "UPDATE claims SET validHeight = ? WHERE nodeName = ? AND txID = ? AND txN = ?"
<< it->name << it->outPoint.hash << it->outPoint.n; << it->name << it->outPoint.hash << it->outPoint.n;
base->_db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->name; db << "UPDATE nodes SET hash = NULL WHERE name = ?" << it->name;
} }
return true; return true;
@ -1188,9 +1189,9 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, const uint160
// cache the parent nodes // cache the parent nodes
getMerkleHash(); getMerkleHash();
proof = CClaimTrieProof(); proof = CClaimTrieProof();
auto nodeQuery = base->_db << "SELECT name FROM nodes WHERE " auto nodeQuery = db << "SELECT name FROM nodes WHERE "
"name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " "name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
"SELECT SUBSTR(p, 1, LENGTH(p)) FROM prefix WHERE p != '') SELECT p FROM prefix) " "SELECT SUBSTR(p, 1, LENGTH(p)-1) FROM prefix WHERE p != '') SELECT p FROM prefix) "
"ORDER BY LENGTH(name)" << name; "ORDER BY LENGTH(name)" << name;
for (auto&& row: nodeQuery) { for (auto&& row: nodeQuery) {
CClaimValue claim; CClaimValue claim;
@ -1203,7 +1204,7 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, const uint160
const auto pos = key.size(); const auto pos = key.size();
std::vector<std::pair<unsigned char, uint256>> children; std::vector<std::pair<unsigned char, uint256>> children;
auto childQuery = base->_db << "SELECT name, hash FROM nodes WHERE parent = ?" << key; auto childQuery = db << "SELECT name, hash FROM nodes WHERE parent = ?" << key;
for (auto&& child : childQuery) { for (auto&& child : childQuery) {
std::string childKey; std::string childKey;
uint256 hash; uint256 hash;
@ -1236,9 +1237,9 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, const uint160
} }
bool CClaimTrieCacheBase::findNameForClaim(const std::vector<unsigned char>& claim, CClaimValue& value, std::string& name) { bool CClaimTrieCacheBase::findNameForClaim(const std::vector<unsigned char>& claim, CClaimValue& value, std::string& name) {
auto query = base->_db << "SELECT nodeName, claimId, txID, txN, amount, validHeight, block_height " auto query = db << "SELECT nodeName, claimId, txID, txN, amount, validHeight, block_height "
"FROM claims WHERE SUBSTR(claimID, 1, ?) = ? AND validHeight < ? AND expirationHeight >= ?" "FROM claims WHERE SUBSTR(claimID, 1, ?) = ? AND validHeight < ? AND expirationHeight >= ?"
<< claim.size() + 1 << claim << nNextHeight << nNextHeight; << claim.size() + 1 << claim << nNextHeight << nNextHeight;
auto hit = false; auto hit = false;
for (auto&& row: query) { for (auto&& row: query) {
if (hit) return false; if (hit) return false;
@ -1251,8 +1252,8 @@ bool CClaimTrieCacheBase::findNameForClaim(const std::vector<unsigned char>& cla
void CClaimTrieCacheBase::getNamesInTrie(std::function<void(const std::string&)> callback) void CClaimTrieCacheBase::getNamesInTrie(std::function<void(const std::string&)> callback)
{ {
auto query = base->_db << "SELECT DISTINCT nodeName FROM claims WHERE validHeight < ? AND expirationHeight >= ?" auto query = db << "SELECT DISTINCT nodeName FROM claims WHERE validHeight < ? AND expirationHeight >= ?"
<< nNextHeight << nNextHeight; << nNextHeight << nNextHeight;
for (auto&& row: query) { for (auto&& row: query) {
std::string name; std::string name;
row >> name; row >> name;

20
src/claimtrie.h
View file

@ -289,22 +289,27 @@ struct CClaimSupportToName
const std::vector<CSupportValue> unmatchedSupports; const std::vector<CSupportValue> unmatchedSupports;
}; };
class CClaimTrieCacheBase;
class CClaimTrie class CClaimTrie
{ {
friend CClaimTrieCacheBase;
std::string dbPath;
int nNextHeight;
sqlite::database db; // keep below dbPath
public: public:
const int nProportionalDelayFactor;
CClaimTrie() = delete; CClaimTrie() = delete;
CClaimTrie(CClaimTrie&&) = delete; CClaimTrie(CClaimTrie&&) = delete;
CClaimTrie(const CClaimTrie&) = delete; CClaimTrie(const CClaimTrie&) = delete;
CClaimTrie(bool fMemory, bool fWipe, int height, int proportionalDelayFactor = 32, std::size_t cacheMB=50); CClaimTrie(bool fWipe, int height, int proportionalDelayFactor = 32, std::size_t cacheMB=50);
CClaimTrie& operator=(CClaimTrie&&) = delete; CClaimTrie& operator=(CClaimTrie&&) = delete;
CClaimTrie& operator=(const CClaimTrie&) = delete; CClaimTrie& operator=(const CClaimTrie&) = delete;
bool SyncToDisk(); bool SyncToDisk();
bool empty(); // for tests bool empty(); // for tests
sqlite::database _db;
int nNextHeight = 0;
int nProportionalDelayFactor = 0;
}; };
struct CClaimTrieProofNode struct CClaimTrieProofNode
@ -402,10 +407,11 @@ public:
protected: protected:
CClaimTrie* base; CClaimTrie* base;
bool dirtyNodes; mutable sqlite::database db;
int nNextHeight; // Height of the block that is being worked on, which is
// one greater than the height of the chain's tip
virtual uint256 recursiveComputeMerkleHash(const std::string& name, bool checkOnly); virtual uint256 recursiveComputeMerkleHash(const std::string& name, bool checkOnly);
supportEntryType getSupportsForName(const std::string& name) const; supportEntryType getSupportsForName(const std::string& name) const;
int getDelayForName(const std::string& name) const; int getDelayForName(const std::string& name) const;
@ -415,8 +421,6 @@ protected:
bool deleteNodeIfPossible(const std::string& name, std::string& parent, std::vector<std::string>& claims); bool deleteNodeIfPossible(const std::string& name, std::string& parent, std::vector<std::string>& claims);
void ensureTreeStructureIsUpToDate(); void ensureTreeStructureIsUpToDate();
int nNextHeight; // Height of the block that is being worked on, which is
// one greater than the height of the chain's tip
private: private:
// for unit test // for unit test

34
src/claimtrieforks.cpp
View file

@ -72,10 +72,9 @@ bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
auto extension = Params().GetConsensus().nExtendedClaimExpirationTime - Params().GetConsensus().nOriginalClaimExpirationTime; auto extension = Params().GetConsensus().nExtendedClaimExpirationTime - Params().GetConsensus().nOriginalClaimExpirationTime;
if (!increment) extension = -extension; if (!increment) extension = -extension;
base->_db << "UPDATE claims SET expirationHeight = expirationHeight + ? WHERE expirationHeight >= ?" << extension << nNextHeight; db << "UPDATE claims SET expirationHeight = expirationHeight + ? WHERE expirationHeight >= ?" << extension << nNextHeight;
base->_db << "UPDATE supports SET expirationHeight = expirationHeight + ? WHERE expirationHeight >= ?" << extension << nNextHeight; db << "UPDATE supports SET expirationHeight = expirationHeight + ? WHERE expirationHeight >= ?" << extension << nNextHeight;
base->_db << "UPDATE nodes SET hash = NULL"; // recompute all hashes (as there aren't that many at this point) db << "UPDATE nodes SET hash = NULL"; // recompute all hashes (as there aren't that many at this point)
dirtyNodes = true;
return true; return true;
} }
@ -133,19 +132,18 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(bool f
// run the one-time upgrade of all names that need to change // run the one-time upgrade of all names that need to change
// it modifies the (cache) trie as it goes, so we need to grab everything to be modified first // it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
base->_db.define("NORMALIZED", [this](const std::string& str) { return normalizeClaimName(str, true); }); db.define("NORMALIZED", [this](const std::string& str) { return normalizeClaimName(str, true); });
auto query = base->_db << "SELECT NORMALIZED(name) as nn, name, claimID FROM claims HAVING nodeName != nn"; auto query = db << "SELECT NORMALIZED(name) as nn, name, claimID FROM claims HAVING nodeName != nn";
for(auto&& row: query) { for(auto&& row: query) {
std::string newName, oldName; std::string newName, oldName;
uint160 claimID; uint160 claimID;
row >> newName >> oldName >> claimID; row >> newName >> oldName >> claimID;
if (!forward) std::swap(newName, oldName); if (!forward) std::swap(newName, oldName);
base->_db << "UPDATE claims SET nodeName = ? WHERE claimID = ?" << newName << claimID; db << "UPDATE claims SET nodeName = ? WHERE claimID = ?" << newName << claimID;
base->_db << "DELETE FROM nodes WHERE name = ?" << oldName; db << "DELETE FROM nodes WHERE name = ?" << oldName;
base->_db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT DO UPDATE hash = NULL" << newName; db << "INSERT INTO nodes(name) VALUES(?) ON CONFLICT DO UPDATE hash = NULL" << newName;
} }
dirtyNodes = true;
return true; return true;
} }
@ -201,7 +199,7 @@ uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(const std::string& n
if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight) if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
return CClaimTrieCacheNormalizationFork::recursiveComputeMerkleHash(name, checkOnly); return CClaimTrieCacheNormalizationFork::recursiveComputeMerkleHash(name, checkOnly);
auto childQuery = base->_db << "SELECT name, hash FROM nodes WHERE parent = ? ORDER BY name" << name; auto childQuery = db << "SELECT name, hash FROM nodes WHERE parent = ? ORDER BY name" << name;
std::vector<uint256> childHashes; std::vector<uint256> childHashes;
for (auto&& row: childQuery) { for (auto&& row: childQuery) {
@ -215,7 +213,7 @@ uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(const std::string& n
childHashes.push_back(*hash); childHashes.push_back(*hash);
} }
auto claimQuery = base->_db << "SELECT c.txID, c.txN, c.validHeight, c.amount + " auto claimQuery = db << "SELECT c.txID, c.txN, c.validHeight, c.amount + "
"SUM(SELECT s.amount FROM supports s WHERE s.supportedClaimID = c.claimID " "SUM(SELECT s.amount FROM supports s WHERE s.supportedClaimID = c.claimID "
"AND s.validHeight < ? AND s.expirationHeight >= ?) as effectiveAmount" "AND s.validHeight < ? AND s.expirationHeight >= ?) as effectiveAmount"
"FROM claims c WHERE c.nodeName = ? AND c.validHeight < ? AND c.expirationHeight >= ? " "FROM claims c WHERE c.nodeName = ? AND c.validHeight < ? AND c.expirationHeight >= ? "
@ -235,7 +233,7 @@ uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(const std::string& n
auto computedHash = Hash(left.begin(), left.end(), right.begin(), right.end()); auto computedHash = Hash(left.begin(), left.end(), right.begin(), right.end());
if (!checkOnly) if (!checkOnly)
base->_db << "UPDATE nodes SET hash = ? WHERE name = ?" << computedHash << name; db << "UPDATE nodes SET hash = ? WHERE name = ?" << computedHash << name;
return computedHash; return computedHash;
} }
@ -308,16 +306,16 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uin
// cache the parent nodes // cache the parent nodes
getMerkleHash(); getMerkleHash();
proof = CClaimTrieProof(); proof = CClaimTrieProof();
auto nodeQuery = base->_db << "SELECT name FROM nodes WHERE " auto nodeQuery = db << "SELECT name FROM nodes WHERE "
"name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " "name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
"SELECT SUBSTR(p, 1, LENGTH(p)) FROM prefix WHERE p != '') SELECT p FROM prefix) " "SELECT SUBSTR(p, 1, LENGTH(p)-1) FROM prefix WHERE p != '') SELECT p FROM prefix) "
"ORDER BY LENGTH(name)" << name; "ORDER BY LENGTH(name)" << name;
for (auto&& row: nodeQuery) { for (auto&& row: nodeQuery) {
std::string key;; std::string key;;
row >> key; row >> key;
std::vector<uint256> childHashes; std::vector<uint256> childHashes;
uint32_t nextCurrentIdx = 0; uint32_t nextCurrentIdx = 0;
auto childQuery = base->_db << "SELECT name, hash FROM nodes WHERE parent = ?" << key; auto childQuery = db << "SELECT name, hash FROM nodes WHERE parent = ?" << key;
for (auto&& child : childQuery) { for (auto&& child : childQuery) {
std::string childKey; std::string childKey;
uint256 childHash; uint256 childHash;
@ -367,14 +365,14 @@ void CClaimTrieCacheHashFork::initializeIncrement()
CClaimTrieCacheNormalizationFork::initializeIncrement(); CClaimTrieCacheNormalizationFork::initializeIncrement();
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests) // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
if (nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1) if (nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
base->_db << "UPDATE nodes SET hash = NULL"; db << "UPDATE nodes SET hash = NULL";
} }
bool CClaimTrieCacheHashFork::finalizeDecrement() bool CClaimTrieCacheHashFork::finalizeDecrement()
{ {
auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement(); auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement();
if (ret && nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1) if (ret && nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
base->_db << "UPDATE nodes SET hash = NULL"; db << "UPDATE nodes SET hash = NULL";
return ret; return ret;
} }

2
src/init.cpp
View file

@ -1463,7 +1463,7 @@ bool AppInitMain()
int64_t trieCacheMB = gArgs.GetArg("-claimtriecache", nDefaultDbCache); int64_t trieCacheMB = gArgs.GetArg("-claimtriecache", nDefaultDbCache);
trieCacheMB = std::min(trieCacheMB, nMaxDbCache); trieCacheMB = std::min(trieCacheMB, nMaxDbCache);
trieCacheMB = std::max(trieCacheMB, nMinDbCache); trieCacheMB = std::max(trieCacheMB, nMinDbCache);
pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState, 32, trieCacheMB); pclaimTrie = new CClaimTrie(fReindex || fReindexChainState, 32, trieCacheMB);
if (fReset) { if (fReset) {
pblocktree->WriteReindexing(true); pblocktree->WriteReindexing(true);

2
src/rpc/claimtrie.cpp
View file

@ -653,7 +653,7 @@ UniValue getnameproof(const JSONRPCRequest& request)
CCoinsViewCache coinsCache(pcoinsTip.get()); CCoinsViewCache coinsCache(pcoinsTip.get());
CClaimTrieCache trieCache(pclaimTrie); CClaimTrieCache trieCache(pclaimTrie);
int validHeight = pclaimTrie->nNextHeight - 1; int validHeight = chainActive.Tip()->nHeight;
if (request.params.size() > 1) { if (request.params.size() > 1) {
CBlockIndex* pblockIndex = BlockHashIndex(ParseHashV(request.params[1], T_BLOCKHASH " (optional parameter 2)")); CBlockIndex* pblockIndex = BlockHashIndex(ParseHashV(request.params[1], T_BLOCKHASH " (optional parameter 2)"));
RollBackTo(pblockIndex, coinsCache, trieCache); RollBackTo(pblockIndex, coinsCache, trieCache);

13
src/test/claimtriecache_tests.cpp
View file

@ -273,18 +273,23 @@ BOOST_AUTO_TEST_CASE(trie_stays_consistent_test)
"goodness", "goodnight", "goodnatured", "goods", "go", "goody", "goo" "goodness", "goodnight", "goodnatured", "goods", "go", "goody", "goo"
}; };
CClaimTrie trie(true, false, 1); CClaimTrie trie(false, 1);
CClaimTrieCacheTest cache(&trie); CClaimTrieCacheTest cache(&trie);
CClaimValue value; CClaimValue value;
for (auto& name: names) for (auto& name: names) {
value.outPoint.hash = Hash(name.begin(), name.end());
value.outPoint.n = 0;
value.claimId = ClaimIdHash(value.outPoint.hash, 0);
BOOST_CHECK(cache.insertClaimIntoTrie(name, value)); BOOST_CHECK(cache.insertClaimIntoTrie(name, value));
}
cache.flush(); cache.flush();
BOOST_CHECK(cache.checkConsistency()); BOOST_CHECK(cache.checkConsistency());
for (auto& name: names) { for (auto& name: names) {
BOOST_CHECK(cache.removeClaimFromTrie(name, COutPoint())); auto hash = Hash(name.begin(), name.end());
BOOST_CHECK(cache.removeClaimFromTrie(name, COutPoint(hash, 0)));
cache.flush(); cache.flush();
BOOST_CHECK(cache.checkConsistency()); BOOST_CHECK(cache.checkConsistency());
} }
@ -299,7 +304,7 @@ BOOST_AUTO_TEST_CASE(takeover_workaround_triggers)
BOOST_SCOPE_EXIT(&consensus, currentMax) { consensus.nMaxTakeoverWorkaroundHeight = currentMax; } BOOST_SCOPE_EXIT(&consensus, currentMax) { consensus.nMaxTakeoverWorkaroundHeight = currentMax; }
BOOST_SCOPE_EXIT_END BOOST_SCOPE_EXIT_END
CClaimTrie trie(true, false, 1); CClaimTrie trie(false, 1);
CClaimTrieCacheTest cache(&trie); CClaimTrieCacheTest cache(&trie);
insertUndoType icu, isu; claimQueueRowType ecu; supportQueueRowType esu; insertUndoType icu, isu; claimQueueRowType ecu; supportQueueRowType esu;

18
src/test/claimtriefixture.cpp
View file

@ -307,35 +307,35 @@ void ClaimTrieChainFixture::DecrementBlocks()
DecrementBlocks(chainActive.Height() - mark); DecrementBlocks(chainActive.Height() - mark);
} }
bool ClaimTrieChainFixture::queueEmpty() bool ClaimTrieChainFixture::queueEmpty() const
{ {
int64_t count; int64_t count;
base->_db << "SELECT COUNT(*) FROM claims WHERE validHeight >= ?" << nNextHeight >> count; db << "SELECT COUNT(*) FROM claims WHERE validHeight >= ?" << nNextHeight >> count;
return count == 0; return count == 0;
} }
bool ClaimTrieChainFixture::expirationQueueEmpty() bool ClaimTrieChainFixture::expirationQueueEmpty() const
{ {
int64_t count; int64_t count;
base->_db << "SELECT COUNT(*) FROM claims WHERE expirationHeight >= ?" << nNextHeight >> count; db << "SELECT COUNT(*) FROM claims WHERE expirationHeight >= ?" << nNextHeight >> count;
return count == 0; return count == 0;
} }
bool ClaimTrieChainFixture::supportEmpty() bool ClaimTrieChainFixture::supportEmpty() const
{ {
int64_t count; int64_t count;
base->_db << "SELECT COUNT(*) FROM supports" >> count; db << "SELECT COUNT(*) FROM supports" >> count;
return count == 0; return count == 0;
} }
bool ClaimTrieChainFixture::supportQueueEmpty() bool ClaimTrieChainFixture::supportQueueEmpty() const
{ {
int64_t count; int64_t count;
base->_db << "SELECT COUNT(*) FROM supports WHERE validHeight >= ?" << nNextHeight >> count; db << "SELECT COUNT(*) FROM supports WHERE validHeight >= ?" << nNextHeight >> count;
return count == 0; return count == 0;
} }
int ClaimTrieChainFixture::proportionalDelayFactor() int ClaimTrieChainFixture::proportionalDelayFactor() const
{ {
return base->nProportionalDelayFactor; return base->nProportionalDelayFactor;
} }

10
src/test/claimtriefixture.h
View file

@ -93,15 +93,15 @@ struct ClaimTrieChainFixture: public CClaimTrieCache
// decrement back to last mark // decrement back to last mark
void DecrementBlocks(); void DecrementBlocks();
bool queueEmpty(); bool queueEmpty() const;
bool expirationQueueEmpty(); bool expirationQueueEmpty() const;
bool supportEmpty(); bool supportEmpty() const;
bool supportQueueEmpty(); bool supportQueueEmpty() const;
int proportionalDelayFactor(); int proportionalDelayFactor() const;
bool getClaimById(const uint160& claimId, std::string& name, CClaimValue& value); bool getClaimById(const uint160& claimId, std::string& name, CClaimValue& value);

4
src/test/test_bitcoin.cpp
View file

@ -83,7 +83,7 @@ std::ostream& operator<<(std::ostream& os, const CSupportValue& support)
} }
BasicTestingSetup::BasicTestingSetup(const std::string& chainName) BasicTestingSetup::BasicTestingSetup(const std::string& chainName)
: m_path_root(fs::temp_directory_path() / "test_bitcoin" / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30)))) : m_path_root(fs::temp_directory_path() / "test_lbrycrd" / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30))))
{ {
// for debugging: // for debugging:
if (boost::unit_test::runtime_config::get<boost::unit_test::log_level>(boost::unit_test::runtime_config::btrt_log_level) if (boost::unit_test::runtime_config::get<boost::unit_test::log_level>(boost::unit_test::runtime_config::btrt_log_level)
@ -138,7 +138,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha
pblocktree.reset(new CBlockTreeDB(1 << 20, true)); pblocktree.reset(new CBlockTreeDB(1 << 20, true));
pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true));
pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get()));
pclaimTrie = new CClaimTrie(true, false, 1); pclaimTrie = new CClaimTrie(true, 1);
if (!LoadGenesisBlock(chainparams)) { if (!LoadGenesisBlock(chainparams)) {
throw std::runtime_error("LoadGenesisBlock failed."); throw std::runtime_error("LoadGenesisBlock failed.");
} }