LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

separate claim from children storage

Browse source
This commit is contained in:
Brannon King 2019-08-24 18:43:56 -06:00
parent c6e267e970
commit 4a5d310fd3
6 changed files with 217 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

266
src/claimtrie.cpp
View file

@ -127,7 +127,7 @@ void CClaimTrieData::reorderClaims(const supportEntryType& supports)
CClaimTrie::CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor) CClaimTrie::CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor)
{ {
nProportionalDelayFactor = proportionalDelayFactor; nProportionalDelayFactor = proportionalDelayFactor;
db.reset(new CDBWrapper(GetDataDir() / "claimtrie", 200 * 1024 * 1024, fMemory, fWipe, false)); db.reset(new CDBWrapper(GetDataDir() / "claimtrie", 400 * 1024 * 1024, fMemory, fWipe, false));
} }
bool CClaimTrie::SyncToDisk() bool CClaimTrie::SyncToDisk()
@ -221,9 +221,8 @@ bool CClaimTrieCacheBase::haveClaim(const std::string& name, const COutPoint& ou
return true; return true;
if (it || nodesToDelete.count(name)) if (it || nodesToDelete.count(name))
return false; return false;
CClaimTrieDataNode node; CClaimTrieData data;
node.childrenSerialization = false; return base->find(name, data) && data.haveClaim(outPoint);
return base->find(name, node) && node.data.haveClaim(outPoint);
} }
bool CClaimTrieCacheBase::haveSupport(const std::string& name, const COutPoint& outPoint) const bool CClaimTrieCacheBase::haveSupport(const std::string& name, const COutPoint& outPoint) const
@ -276,12 +275,19 @@ bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COut
return haveInQueue<CSupportValue>(name, outPoint, nValidAtHeight); return haveInQueue<CSupportValue>(name, outPoint, nValidAtHeight);
} }
void CClaimTrie::recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<void(const std::string&, const CClaimTrieDataNode&)> function) const { void CClaimTrie::recurseNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<recurseNodesCB> function) const {
function(name, current); CClaimTrieData data;
if (!find(name, data))
data = {};
data.hash = current.hash;
data.flags |= current.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
function(name, data, current.children);
for (auto& child: current.children) { for (auto& child: current.children) {
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find(child.second, node)) if (find(name + child, node))
recurseAllHashedNodes(name + child.first, node, function); recurseNodes(name + child, node, function);
} }
} }
@ -290,8 +296,8 @@ std::size_t CClaimTrie::getTotalNamesInTrie() const
std::size_t count = 0; std::size_t count = 0;
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find("", node)) if (find("", node))
recurseAllHashedNodes("", node, [&count](const std::string&, const CClaimTrieDataNode& node) { recurseNodes("", node, [&count](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
count += !node.data.empty(); count += !data.empty();
}); });
return count; return count;
} }
@ -301,8 +307,9 @@ std::size_t CClaimTrie::getTotalClaimsInTrie() const
std::size_t count = 0; std::size_t count = 0;
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find("", node)) if (find("", node))
recurseAllHashedNodes("", node, [&count](const std::string&, const CClaimTrieDataNode& node) { recurseNodes("", node, [&count]
count += node.data.claims.size(); (const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
count += data.claims.size();
}); });
return count; return count;
} }
@ -310,11 +317,11 @@ std::size_t CClaimTrie::getTotalClaimsInTrie() const
CAmount CClaimTrie::getTotalValueOfClaimsInTrie(bool fControllingOnly) const CAmount CClaimTrie::getTotalValueOfClaimsInTrie(bool fControllingOnly) const
{ {
CAmount value_in_subtrie = 0; CAmount value_in_subtrie = 0;
std::size_t count = 0;
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find("", node)) if (find("", node))
recurseAllHashedNodes("", node, [&value_in_subtrie, fControllingOnly](const std::string&, const CClaimTrieDataNode& node) { recurseNodes("", node, [&value_in_subtrie, fControllingOnly]
for (const auto& claim : node.data.claims) { (const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
for (const auto &claim : data.claims) {
value_in_subtrie += claim.nAmount; value_in_subtrie += claim.nAmount;
if (fControllingOnly) if (fControllingOnly)
break; break;
@ -330,9 +337,8 @@ bool CClaimTrieCacheBase::getInfoForName(const std::string& name, CClaimValue& c
return true; return true;
if (it || nodesToDelete.count(name)) if (it || nodesToDelete.count(name))
return false; return false;
CClaimTrieDataNode node; CClaimTrieData claims;
node.childrenSerialization = false; return base->find(name, claims) && claims.getBestClaim(claim);
return base->find(name, node) && node.data.getBestClaim(claim);
} }
CClaimsForNameType CClaimTrieCacheBase::getClaimsForName(const std::string& name) const CClaimsForNameType CClaimTrieCacheBase::getClaimsForName(const std::string& name) const
@ -341,15 +347,14 @@ CClaimsForNameType CClaimTrieCacheBase::getClaimsForName(const std::string& name
int nLastTakeoverHeight = 0; int nLastTakeoverHeight = 0;
auto supports = getSupportsForName(name); auto supports = getSupportsForName(name);
CClaimTrieDataNode node; CClaimTrieData data;
node.childrenSerialization = false;
if (auto it = nodesToAddOrUpdate.find(name)) { if (auto it = nodesToAddOrUpdate.find(name)) {
claims = it->claims; claims = it->claims;
nLastTakeoverHeight = it->nHeightOfLastTakeover; nLastTakeoverHeight = it->nHeightOfLastTakeover;
} }
else if (!nodesToDelete.count(name) && base->find(name, node)) { else if (!nodesToDelete.count(name) && base->find(name, data)) {
claims = node.data.claims; claims = data.claims;
nLastTakeoverHeight = node.data.nHeightOfLastTakeover; nLastTakeoverHeight = data.nHeightOfLastTakeover;
} }
return {std::move(claims), std::move(supports), nLastTakeoverHeight, name}; return {std::move(claims), std::move(supports), nLastTakeoverHeight, name};
} }
@ -390,35 +395,10 @@ void completeHash(uint256& partialHash, const std::string& key, std::size_t to)
template <typename T> template <typename T>
using iCbType = std::function<void(T&)>; using iCbType = std::function<void(T&)>;
template <typename TIterator>
uint256 recursiveMerkleHash(TIterator& it, const iCbType<TIterator>& process, const iCbType<TIterator>& verify = {})
{
std::vector<uint8_t> vchToHash;
const auto pos = it.key().size();
for (auto& child : it.children()) {
process(child);
auto& key = child.key();
auto hash = child->hash;
completeHash(hash, key, pos);
vchToHash.push_back(key[pos]);
vchToHash.insert(vchToHash.end(), hash.begin(), hash.end());
}
CClaimValue claim;
if (it->getBestClaim(claim)) {
uint256 valueHash = getValueHash(claim.outPoint, it->nHeightOfLastTakeover);
vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
} else if (verify) {
verify(it);
}
return Hash(vchToHash.begin(), vchToHash.end());
}
bool CClaimTrie::checkConsistency(const uint256& rootHash) const bool CClaimTrie::checkConsistency(const uint256& rootHash) const
{ {
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (!find("", node) || node.data.hash != rootHash) { if (!find("", node) || node.hash != rootHash) {
if (rootHash == one) if (rootHash == one)
return true; return true;
@ -426,30 +406,29 @@ bool CClaimTrie::checkConsistency(const uint256& rootHash) const
} }
bool success = true; bool success = true;
recurseAllHashedNodes("", node, [&success, this](const std::string& name, const CClaimTrieDataNode& node) { recurseNodes("", node, [&success, this](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
if (!success) return; if (!success) return;
success &= contains(name);
std::vector<uint8_t> vchToHash; std::vector<uint8_t> vchToHash;
const auto pos = name.size(); const auto pos = name.size();
for (auto& child : node.children) { for (auto &child : children) {
auto key = name + child.first; auto key = name + child;
auto hash = child.second; CClaimTrieDataNode node;
success &= find(key, node);
auto hash = node.hash;
completeHash(hash, key, pos); completeHash(hash, key, pos);
vchToHash.push_back(key[pos]); vchToHash.push_back(key[pos]);
vchToHash.insert(vchToHash.end(), hash.begin(), hash.end()); vchToHash.insert(vchToHash.end(), hash.begin(), hash.end());
} }
CClaimValue claim; CClaimValue claim;
if (node.data.getBestClaim(claim)) { if (data.getBestClaim(claim)) {
uint256 valueHash = getValueHash(claim.outPoint, node.data.nHeightOfLastTakeover); uint256 valueHash = getValueHash(claim.outPoint, data.nHeightOfLastTakeover);
vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end()); vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
} else { } else {
success &= !node.children.empty(); // we disallow leaf nodes without claims success &= !children.empty(); // we disallow leaf nodes without claims
} }
success &= data.hash == Hash(vchToHash.begin(), vchToHash.end());
success &= node.data.hash == Hash(vchToHash.begin(), vchToHash.end());
}); });
return success; return success;
@ -467,21 +446,22 @@ std::vector<std::pair<std::string, CClaimTrieDataNode>> CClaimTrie::nodes(const
while (!node.children.empty()) { while (!node.children.empty()) {
// auto it = node.children.lower_bound(partialKey); // for using a std::map // auto it = node.children.lower_bound(partialKey); // for using a std::map
auto it = std::lower_bound(node.children.begin(), node.children.end(), std::make_pair(partialKey, uint256())); auto it = std::lower_bound(node.children.begin(), node.children.end(), partialKey);
if (it != node.children.end() && it->first == partialKey) { if (it != node.children.end() && *it == partialKey) {
// we're completely done // we're completely done
if (find(it->second, node)) if (find(key, node))
ret.emplace_back(key, node); ret.emplace_back(key, node);
break; break;
} }
if (it != node.children.begin()) --it; if (it != node.children.begin()) --it;
const auto count = match(partialKey, it->first); const auto count = match(partialKey, *it);
if (count != it->first.size()) break; if (count != it->size()) break;
if (count == partialKey.size()) break; if (count == partialKey.size()) break;
partialKey = partialKey.substr(count); partialKey = partialKey.substr(count);
if (find(it->second, node)) auto frontKey = key.substr(0, key.size() - partialKey.size());
ret.emplace_back(key.substr(0, key.size() - partialKey.size()), node); if (find(frontKey, node))
ret.emplace_back(frontKey, node);
else break; else break;
} }
@ -489,7 +469,7 @@ std::vector<std::pair<std::string, CClaimTrieDataNode>> CClaimTrie::nodes(const
} }
bool CClaimTrie::contains(const std::string &key) const { bool CClaimTrie::contains(const std::string &key) const {
return db->Exists(std::make_pair(TRIE_NODE_BY_NAME, key)); return db->Exists(std::make_pair(TRIE_NODE_CHILDREN, key));
} }
bool CClaimTrie::empty() const { bool CClaimTrie::empty() const {
@ -497,15 +477,11 @@ bool CClaimTrie::empty() const {
} }
bool CClaimTrie::find(const std::string& key, CClaimTrieDataNode &node) const { bool CClaimTrie::find(const std::string& key, CClaimTrieDataNode &node) const {
uint256 hash; return db->Read(std::make_pair(TRIE_NODE_CHILDREN, key), node);
if (!db->Read(std::make_pair(TRIE_NODE_BY_NAME, key), hash))
return false;
auto found = find(hash, node);
return found;
} }
bool CClaimTrie::find(const uint256 &key, CClaimTrieDataNode &node) const { bool CClaimTrie::find(const std::string& key, CClaimTrieData &data) const {
return db->Read(std::make_pair(TRIE_NODE_BY_HASH, key), node); return db->Read(std::make_pair(TRIE_NODE_CLAIMS, key), data);
} }
bool CClaimTrieCacheBase::getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const bool CClaimTrieCacheBase::getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const
@ -555,7 +531,7 @@ bool CClaimTrieCacheBase::flush()
for (const auto& e : claimsToAddToByIdIndex) for (const auto& e : claimsToAddToByIdIndex)
batch.Write(std::make_pair(CLAIM_BY_ID, e.claim.claimId), e); batch.Write(std::make_pair(CLAIM_BY_ID, e.claim.claimId), e);
getMerkleHash(); auto rootHash = getMerkleHash();
std::set<std::string> forDeletion; std::set<std::string> forDeletion;
for (const auto& nodeName : nodesToDelete) { for (const auto& nodeName : nodesToDelete) {
@ -567,21 +543,23 @@ bool CClaimTrieCacheBase::flush()
} }
for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) { for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) {
forDeletion.erase(it.key()); bool removed = forDeletion.erase(it.key());
if (!dirtyNodes.count(it.key())) if (it->flags & CClaimTrieDataFlags::HASH_DIRTY) {
continue;
CClaimTrieDataNode node; CClaimTrieDataNode node;
node.data = it.data(); node.hash = it->hash;
for (auto &child: it.children()) // ordering here is important for (auto &child: it.children()) // ordering here is important
node.children.emplace_back(child.key().substr(it.key().size()), child->hash); node.children.push_back(child.key().substr(it.key().size()));
batch.Write(std::make_pair(TRIE_NODE_BY_HASH, it->hash), node); batch.Write(std::make_pair(TRIE_NODE_CHILDREN, it.key()), node);
batch.Write(std::make_pair(TRIE_NODE_BY_NAME, it.key()), it->hash);
if (removed || (it->flags & CClaimTrieDataFlags::CLAIMS_DIRTY))
batch.Write(std::make_pair(TRIE_NODE_CLAIMS, it.key()), it.data());
}
} }
for (auto& name: forDeletion) { for (auto& name: forDeletion) {
batch.Erase(std::make_pair(TRIE_NODE_BY_NAME, name)); batch.Erase(std::make_pair(TRIE_NODE_CHILDREN, name));
batch.Erase(std::make_pair(TRIE_NODE_CLAIMS, name));
} }
BatchWriteQueue(batch, SUPPORT, supportCache); BatchWriteQueue(batch, SUPPORT, supportCache);
@ -600,6 +578,18 @@ bool CClaimTrieCacheBase::flush()
nodesToAddOrUpdate.height(), nNextHeight, batch.SizeEstimate()); nodesToAddOrUpdate.height(), nNextHeight, batch.SizeEstimate());
} }
auto ret = base->db->WriteBatch(batch); auto ret = base->db->WriteBatch(batch);
// for debugging:
// if (nNextHeight >= 235099)
// {
// g_logger->EnableCategory(BCLog::CLAIMS);
// if (!base->checkConsistency(rootHash)) {
// LogPrintf("Failure with consistency on block height %d\n", nNextHeight);
// dumpToLog(begin());
// assert(false);
// }
// }
clear(); clear();
return ret; return ret;
} }
@ -623,7 +613,7 @@ bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0; base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0;
clear(); clear();
if (tip && (base->db->Exists(std::make_pair(TRIE_NODE, std::string())) || !base->db->Exists(std::make_pair(TRIE_NODE_BY_HASH, tip->hashClaimTrie)))) { if (tip && base->db->Exists(std::make_pair(TRIE_NODE, std::string()))) {
LogPrintf("The claim trie database contains deprecated data and will need to be rebuilt"); LogPrintf("The claim trie database contains deprecated data and will need to be rebuilt");
return false; return false;
} }
@ -643,13 +633,28 @@ int CClaimTrieCacheBase::expirationTime() const
uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it) uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it)
{ {
using iterator = CClaimPrefixTrie::iterator; if (!it->hash.IsNull())
iCbType<iterator> process = [&process](iterator& it) {
if (it->hash.IsNull())
it->hash = recursiveMerkleHash(it, process);
};
process(it);
return it->hash; return it->hash;
std::vector<uint8_t> vchToHash;
const auto pos = it.key().size();
for (auto& child : it.children()) {
auto hash = recursiveComputeMerkleHash(child);
auto& key = child.key();
completeHash(hash, key, pos);
vchToHash.push_back(key[pos]);
vchToHash.insert(vchToHash.end(), hash.begin(), hash.end());
}
CClaimValue claim;
if (it->getBestClaim(claim)) {
uint256 valueHash = getValueHash(claim.outPoint, it->nHeightOfLastTakeover);
vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
}
auto ret = Hash(vchToHash.begin(), vchToHash.end());
it->hash = ret;
return ret;
} }
uint256 CClaimTrieCacheBase::getMerkleHash() uint256 CClaimTrieCacheBase::getMerkleHash()
@ -659,9 +664,8 @@ uint256 CClaimTrieCacheBase::getMerkleHash()
return recursiveComputeMerkleHash(it); return recursiveComputeMerkleHash(it);
if (nodesToDelete.empty() && nodesAlreadyCached.empty()) { if (nodesToDelete.empty() && nodesAlreadyCached.empty()) {
CClaimTrieDataNode node; CClaimTrieDataNode node;
node.childrenSerialization = false;
if (base->find("", node)) if (base->find("", node))
return node.data.hash; // it may be valuable to have base cache its current root hash return node.hash; // it may be valuable to have base cache its current root hash
} }
return one; // we have no data or we deleted everything return one; // we have no data or we deleted everything
} }
@ -689,16 +693,25 @@ CClaimPrefixTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& nam
for (auto& node: nodes) { for (auto& node: nodes) {
if (nodesAlreadyCached.insert(node.first).second) { if (nodesAlreadyCached.insert(node.first).second) {
// do not insert nodes that are already present // do not insert nodes that are already present
nodesToAddOrUpdate.insert(node.first, node.second.data); CClaimTrieData data;
if (!base->find(node.first, data))
data = {};
data.hash = node.second.hash;
data.flags = node.second.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
nodesToAddOrUpdate.insert(node.first, data);
} }
for (auto& child : node.second.children) { for (auto& child : node.second.children) {
auto childKey = node.first + child.first; auto childKey = node.first + child;
if (nodesAlreadyCached.insert(childKey).second) { if (nodesAlreadyCached.insert(childKey).second) {
CClaimTrieData childData;
if (!base->find(childKey, childData))
childData = {};
CClaimTrieDataNode childNode; CClaimTrieDataNode childNode;
childNode.childrenSerialization = false; if (base->find(childKey, childNode)) { // TODO: can we eliminate this expensive lookup?
if (base->find(child.second, childNode)) { childData.hash = childNode.hash;
nodesToAddOrUpdate.insert(childKey, childNode.data); childData.flags = childNode.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
} }
nodesToAddOrUpdate.insert(childKey, childData);
} }
} }
} }
@ -706,6 +719,8 @@ CClaimPrefixTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& nam
auto it = nodesToAddOrUpdate.find(name); auto it = nodesToAddOrUpdate.find(name);
if (!it && create) { if (!it && create) {
it = nodesToAddOrUpdate.insert(name, CClaimTrieData{}); it = nodesToAddOrUpdate.insert(name, CClaimTrieData{});
// if (it.hasChildren()) any children should be in the trie (not base alone)
// it->flags |= CClaimTrieDataFlags::POTENTIAL_CHILDREN;
confirmTakeoverWorkaroundNeeded(name); confirmTakeoverWorkaroundNeeded(name);
} }
@ -726,12 +741,11 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16
std::tie(claimId, takeoverHeight) = cit->second; std::tie(claimId, takeoverHeight) = cit->second;
return true; return true;
} }
CClaimTrieDataNode data; CClaimTrieData data;
data.childrenSerialization = false;
if (base->find(name, data)) { if (base->find(name, data)) {
takeoverHeight = data.data.nHeightOfLastTakeover; takeoverHeight = data.nHeightOfLastTakeover;
CClaimValue claim; CClaimValue claim;
if (data.data.getBestClaim(claim)) { if (data.getBestClaim(claim)) {
claimId = claim.claimId; claimId = claim.claimId;
return true; return true;
} }
@ -742,8 +756,10 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16
void CClaimTrieCacheBase::markAsDirty(const std::string& name, bool fCheckTakeover) void CClaimTrieCacheBase::markAsDirty(const std::string& name, bool fCheckTakeover)
{ {
for (auto& node : nodesToAddOrUpdate.nodes(name)) { for (auto& node : nodesToAddOrUpdate.nodes(name)) {
dirtyNodes.insert(node.key()); node->flags |= CClaimTrieDataFlags::HASH_DIRTY;
node->hash.SetNull(); node->hash.SetNull();
if (node.key() == name)
node->flags |= CClaimTrieDataFlags::CLAIMS_DIRTY;
} }
if (fCheckTakeover) if (fCheckTakeover)
@ -1022,8 +1038,7 @@ void CClaimTrieCacheBase::dumpToLog(CClaimPrefixTrie::const_iterator it, bool di
if (diffFromBase) { if (diffFromBase) {
CClaimTrieDataNode node; CClaimTrieDataNode node;
node.childrenSerialization = false; if (base->find(it.key(), node) && node.hash == it->hash)
if (base->find(it.key(), node) && node.data.hash == it->hash)
return; return;
} }
@ -1342,10 +1357,9 @@ int CClaimTrieCacheBase::getNumBlocksOfContinuousOwnership(const std::string& na
return nNextHeight - it->nHeightOfLastTakeover; return nNextHeight - it->nHeightOfLastTakeover;
if (it) // we specifically ignore deleted nodes here to allow this to fall into the base lookup in that scenario if (it) // we specifically ignore deleted nodes here to allow this to fall into the base lookup in that scenario
return 0; return 0;
CClaimTrieDataNode node; CClaimTrieData data;
node.childrenSerialization = false; if (base->find(name, data) && !data.empty())
if (base->find(name, node) && !node.data.empty()) return nNextHeight - data.nHeightOfLastTakeover;
return nNextHeight - node.data.nHeightOfLastTakeover;
return 0; return 0;
} }
@ -1375,7 +1389,6 @@ bool CClaimTrieCacheBase::clear()
{ {
nodesToAddOrUpdate.clear(); nodesToAddOrUpdate.clear();
claimsToAddToByIdIndex.clear(); claimsToAddToByIdIndex.clear();
dirtyNodes.clear();
supportCache.clear(); supportCache.clear();
nodesToDelete.clear(); nodesToDelete.clear();
claimsToDeleteFromByIdIndex.clear(); claimsToDeleteFromByIdIndex.clear();
@ -1442,3 +1455,34 @@ bool CClaimTrieCacheBase::getProofForName(const std::string& name, CClaimTriePro
proof = CClaimTrieProof(std::move(nodes), fNameHasValue, outPoint, nHeightOfLastTakeover); proof = CClaimTrieProof(std::move(nodes), fNameHasValue, outPoint, nHeightOfLastTakeover);
return true; return true;
} }
void CClaimTrieCacheBase::recurseNodes(const std::string &name,
std::function<void(const std::string &, const CClaimTrieData &)> function) const {
std::function<CClaimTrie::recurseNodesCB> baseFunction = [this, &function]
(const std::string& name, const CClaimTrieData& data, const std::vector<std::string>&) {
if (nodesToDelete.find(name) == nodesToDelete.end())
function(name, data);
};
if (empty()) {
CClaimTrieDataNode node;
if (base->find(name, node))
base->recurseNodes(name, node, baseFunction);
}
else {
for (auto it = begin(); it != end(); ++it) {
function(it.key(), it.data());
if ((it->flags & CClaimTrieDataFlags::POTENTIAL_CHILDREN) && !it.hasChildren()) {
CClaimTrieDataNode node;
if (base->find(it.key(), node))
for (auto& partialKey: node.children) {
auto childKey = it.key() + partialKey;
CClaimTrieDataNode childNode;
if (base->find(childKey, childNode))
base->recurseNodes(childKey, childNode, baseFunction);
}
}
}
}
}

35
src/claimtrie.h
View file

@ -19,8 +19,8 @@
// leveldb keys // leveldb keys
#define TRIE_NODE 'n' // deprecated #define TRIE_NODE 'n' // deprecated
#define TRIE_NODE_BY_HASH 'h' #define TRIE_NODE_CHILDREN 'b'
#define TRIE_NODE_BY_NAME 'g' #define TRIE_NODE_CLAIMS 'c'
#define CLAIM_BY_ID 'i' #define CLAIM_BY_ID 'i'
#define CLAIM_QUEUE_ROW 'r' #define CLAIM_QUEUE_ROW 'r'
#define CLAIM_QUEUE_NAME_ROW 'm' #define CLAIM_QUEUE_NAME_ROW 'm'
@ -136,12 +136,21 @@ struct CSupportValue
typedef std::vector<CClaimValue> claimEntryType; typedef std::vector<CClaimValue> claimEntryType;
typedef std::vector<CSupportValue> supportEntryType; typedef std::vector<CSupportValue> supportEntryType;
enum CClaimTrieDataFlags: uint32_t {
HASH_DIRTY = 1U,
CLAIMS_DIRTY = 2U,
POTENTIAL_CHILDREN = 4U, // existing on disk
};
struct CClaimTrieData struct CClaimTrieData
{ {
uint256 hash;
claimEntryType claims; claimEntryType claims;
int nHeightOfLastTakeover = 0; int nHeightOfLastTakeover = 0;
// non-serialized data:
uint256 hash;
uint32_t flags = 0;
CClaimTrieData() = default; CClaimTrieData() = default;
CClaimTrieData(CClaimTrieData&&) = default; CClaimTrieData(CClaimTrieData&&) = default;
CClaimTrieData(const CClaimTrieData&) = default; CClaimTrieData(const CClaimTrieData&) = default;
@ -159,14 +168,13 @@ struct CClaimTrieData
template <typename Stream, typename Operation> template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) inline void SerializationOp(Stream& s, Operation ser_action)
{ {
READWRITE(hash);
READWRITE(claims); READWRITE(claims);
READWRITE(nHeightOfLastTakeover); READWRITE(nHeightOfLastTakeover);
} }
bool operator==(const CClaimTrieData& other) const bool operator==(const CClaimTrieData& other) const
{ {
return hash == other.hash && nHeightOfLastTakeover == other.nHeightOfLastTakeover && claims == other.claims; return nHeightOfLastTakeover == other.nHeightOfLastTakeover && claims == other.claims;
} }
bool operator!=(const CClaimTrieData& other) const bool operator!=(const CClaimTrieData& other) const
@ -181,11 +189,10 @@ struct CClaimTrieData
}; };
struct CClaimTrieDataNode { struct CClaimTrieDataNode {
CClaimTrieData data; uint256 hash;
// we're using a vector to avoid RAM thrashing and for faster serialization ops. // we're using a vector to avoid RAM thrashing and for faster serialization ops.
// We're assuming its data is inserted in order and never modified. // We're assuming its data is inserted in order and never modified.
std::vector<std::pair<std::string, uint256>> children; std::vector<std::string> children;
bool childrenSerialization = true;
CClaimTrieDataNode() = default; CClaimTrieDataNode() = default;
CClaimTrieDataNode(CClaimTrieDataNode&&) = default; CClaimTrieDataNode(CClaimTrieDataNode&&) = default;
@ -198,8 +205,7 @@ struct CClaimTrieDataNode {
template <typename Stream, typename Operation> template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) inline void SerializationOp(Stream& s, Operation ser_action)
{ {
READWRITE(data); READWRITE(hash);
if (childrenSerialization) // wanting constexpr but hoping the compiler is smart enough anyway
READWRITE(children); READWRITE(children);
} }
}; };
@ -346,11 +352,13 @@ public:
bool contains(const std::string& key) const; bool contains(const std::string& key) const;
bool empty() const; bool empty() const;
bool find(const uint256& key, CClaimTrieDataNode& node) const;
bool find(const std::string& key, CClaimTrieDataNode& node) const; bool find(const std::string& key, CClaimTrieDataNode& node) const;
bool find(const std::string& key, CClaimTrieData& claims) const;
std::vector<std::pair<std::string, CClaimTrieDataNode>> nodes(const std::string& key) const; std::vector<std::pair<std::string, CClaimTrieDataNode>> nodes(const std::string& key) const;
void recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<void(const std::string&, const CClaimTrieDataNode&)> function) const;
using recurseNodesCB = void(const std::string&, const CClaimTrieData&, const std::vector<std::string>&);
void recurseNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<recurseNodesCB> function) const;
}; };
struct CClaimTrieProofNode struct CClaimTrieProofNode
@ -473,6 +481,8 @@ public:
void dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase = true) const; void dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase = true) const;
virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const; virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
void recurseNodes(const std::string& name, std::function<void(const std::string&, const CClaimTrieData&)> function) const;
protected: protected:
CClaimTrie* base; CClaimTrie* base;
CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
@ -523,7 +533,6 @@ private:
std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
std::unordered_map<std::string, bool> takeoverWorkaround; std::unordered_map<std::string, bool> takeoverWorkaround;
std::unordered_set<std::string> removalWorkaround; std::unordered_set<std::string> removalWorkaround;
std::unordered_set<std::string> dirtyNodes;
bool shouldUseTakeoverWorkaround(const std::string& key) const; bool shouldUseTakeoverWorkaround(const std::string& key) const;
void addTakeoverWorkaroundPotential(const std::string& key); void addTakeoverWorkaroundPotential(const std::string& key);

2
src/claimtrieforks.cpp
View file

@ -164,7 +164,7 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insert
boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator()); boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) { for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
std::pair<uint8_t, std::string> key; std::pair<uint8_t, std::string> key;
if (!pcursor->GetKey(key) || key.first != TRIE_NODE_BY_NAME) if (!pcursor->GetKey(key) || key.first != TRIE_NODE_CHILDREN)
continue; continue;
const auto& name = key.second; const auto& name = key.second;

26
src/rpc/claimtrie.cpp
View file

@ -76,6 +76,7 @@ void RollBackTo(const CBlockIndex* targetIndex, CCoinsViewCache& coinsCache, CCl
if (g_chainstate.DisconnectBlock(block, activeIndex, coinsCache, trieCache) != DisconnectResult::DISCONNECT_OK) if (g_chainstate.DisconnectBlock(block, activeIndex, coinsCache, trieCache) != DisconnectResult::DISCONNECT_OK)
throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Failed to disconnect %s", block.ToString())); throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Failed to disconnect %s", block.ToString()));
} }
trieCache.getMerkleHash(); // update the hash tree
} }
std::string escapeNonUtf8(const std::string& name) std::string escapeNonUtf8(const std::string& name)
@ -175,7 +176,6 @@ static UniValue getclaimsintrie(const JSONRPCRequest& request)
} }
UniValue ret(UniValue::VARR); UniValue ret(UniValue::VARR);
uint256 rootHash;
LOCK(cs_main); LOCK(cs_main);
CCoinsViewCache coinsCache(pcoinsTip.get()); CCoinsViewCache coinsCache(pcoinsTip.get());
@ -185,24 +185,18 @@ static UniValue getclaimsintrie(const JSONRPCRequest& request)
CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)")); CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)"));
RollBackTo(blockIndex, coinsCache, trieCache); RollBackTo(blockIndex, coinsCache, trieCache);
} }
rootHash = trieCache.getMerkleHash();
CClaimTrieDataNode rootNode; trieCache.recurseNodes("", [&ret, &trieCache, &coinsCache] (const std::string& name, const CClaimTrieData& data) {
if (!pclaimTrie->find(rootHash, rootNode))
return ret;
pclaimTrie->recurseAllHashedNodes("", rootNode, [&ret, &trieCache, &coinsCache](const std::string &name,
const CClaimTrieDataNode &node) {
if (ShutdownRequested()) if (ShutdownRequested())
throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested"); throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested");
boost::this_thread::interruption_point(); boost::this_thread::interruption_point();
if (node.data.empty()) if (data.empty())
return; return;
UniValue claims(UniValue::VARR); UniValue claims(UniValue::VARR);
for (auto itClaims = node.data.claims.cbegin(); itClaims != node.data.claims.cend(); ++itClaims) { for (auto itClaims = data.claims.cbegin(); itClaims != data.claims.cend(); ++itClaims) {
UniValue claim(UniValue::VOBJ); UniValue claim(UniValue::VOBJ);
claim.pushKV("claimId", itClaims->claimId.GetHex()); claim.pushKV("claimId", itClaims->claimId.GetHex());
claim.pushKV("txid", itClaims->outPoint.hash.GetHex()); claim.pushKV("txid", itClaims->outPoint.hash.GetHex());
@ -261,8 +255,6 @@ static UniValue getnamesintrie(const JSONRPCRequest& request)
"Result: \n" "Result: \n"
"\"names\" (array) all names in the trie that have claims\n"); "\"names\" (array) all names in the trie that have claims\n");
uint256 rootHash;
{
LOCK(cs_main); LOCK(cs_main);
CCoinsViewCache coinsCache(pcoinsTip.get()); CCoinsViewCache coinsCache(pcoinsTip.get());
@ -272,16 +264,10 @@ static UniValue getnamesintrie(const JSONRPCRequest& request)
CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)")); CBlockIndex *blockIndex = BlockHashIndex(ParseHashV(request.params[0], "blockhash (optional parameter 1)"));
RollBackTo(blockIndex, coinsCache, trieCache); RollBackTo(blockIndex, coinsCache, trieCache);
} }
rootHash = trieCache.getMerkleHash();
}
UniValue ret(UniValue::VARR); UniValue ret(UniValue::VARR);
CClaimTrieDataNode rootNode; trieCache.recurseNodes("", [&ret](const std::string &name, const CClaimTrieData &data) {
if (!pclaimTrie->find(rootHash, rootNode)) if (!data.empty())
return ret;
pclaimTrie->recurseAllHashedNodes("", rootNode, [&ret](const std::string& name, const CClaimTrieDataNode& node) {
if (!node.data.empty())
ret.push_back(escapeNonUtf8(name)); ret.push_back(escapeNonUtf8(name));
if (ShutdownRequested()) if (ShutdownRequested())
throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested"); throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested");

34
src/test/claimtriebranching_tests.cpp
View file

@ -509,6 +509,22 @@ BOOST_AUTO_TEST_CASE(triehash_fuzzer_test)
std::cerr << "Hash: " << fixture.getMerkleHash().GetHex() << std::endl; std::cerr << "Hash: " << fixture.getMerkleHash().GetHex() << std::endl;
} }
#endif #endif
BOOST_AUTO_TEST_CASE(claim_replace_test) {
// no competing bids
ClaimTrieChainFixture fixture;
CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "bass", "one", 1);
CMutableTransaction tx3 = fixture.MakeClaim(fixture.GetCoinbase(), "basso", "two", 1);
fixture.IncrementBlocks(1);
BOOST_CHECK(fixture.is_best_claim("bass", tx1));
fixture.Spend(tx1);
CMutableTransaction tx2 = fixture.MakeClaim(fixture.GetCoinbase(), "bassfisher", "one", 1);
fixture.IncrementBlocks(1);
BOOST_CHECK(pclaimTrie->checkConsistency(fixture.getMerkleHash()));
BOOST_CHECK(!fixture.is_best_claim("bass", tx1));
BOOST_CHECK(fixture.is_best_claim("bassfisher", tx2));
}
/* /*
claims claims
no competing bids no competing bids
@ -2617,18 +2633,6 @@ BOOST_AUTO_TEST_CASE(claimtrienode_serialize_unserialize)
ss >> n2; ss >> n2;
BOOST_CHECK_EQUAL(n1, n2); BOOST_CHECK_EQUAL(n1, n2);
n1.hash.SetHex("0000000000000000000000000000000000000000000000000000000000000001");
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK_EQUAL(n1, n2);
n1.hash.SetHex("a79e8a5b28f7fa5e8836a4b48da9988bdf56ce749f81f413cb754f963a516200");
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK_EQUAL(n1, n2);
CClaimValue v1(COutPoint(uint256S("0000000000000000000000000000000000000000000000000000000000000001"), 0), hash160, 50, 0, 100); CClaimValue v1(COutPoint(uint256S("0000000000000000000000000000000000000000000000000000000000000001"), 0), hash160, 50, 0, 100);
CClaimValue v2(COutPoint(uint256S("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), 1), hash160, 100, 1, 101); CClaimValue v2(COutPoint(uint256S("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), 1), hash160, 100, 1, 101);
@ -4101,9 +4105,9 @@ BOOST_AUTO_TEST_CASE(update_on_support2_test)
CMutableTransaction s2 = fixture.MakeSupport(fixture.GetCoinbase(), tx1, name, 1); CMutableTransaction s2 = fixture.MakeSupport(fixture.GetCoinbase(), tx1, name, 1);
fixture.IncrementBlocks(1); fixture.IncrementBlocks(1);
CClaimTrieDataNode node; CClaimTrieData node;
BOOST_CHECK(pclaimTrie->find(name, node)); BOOST_CHECK(pclaimTrie->find(name, node));
BOOST_CHECK_EQUAL(node.data.nHeightOfLastTakeover, height + 1); BOOST_CHECK_EQUAL(node.nHeightOfLastTakeover, height + 1);
fixture.Spend(s1); fixture.Spend(s1);
fixture.Spend(s2); fixture.Spend(s2);
@ -4111,7 +4115,7 @@ BOOST_AUTO_TEST_CASE(update_on_support2_test)
fixture.IncrementBlocks(1); fixture.IncrementBlocks(1);
BOOST_CHECK(pclaimTrie->find(name, node)); BOOST_CHECK(pclaimTrie->find(name, node));
BOOST_CHECK_EQUAL(node.data.nHeightOfLastTakeover, height + 1); BOOST_CHECK_EQUAL(node.nHeightOfLastTakeover, height + 1);
} }
BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END()

5
src/test/claimtriecache_tests.cpp
View file

@ -200,7 +200,6 @@ BOOST_AUTO_TEST_CASE(basic_insertion_info_test)
CClaimTrieCacheTest ctc(pclaimTrie); CClaimTrieCacheTest ctc(pclaimTrie);
// create and insert claim // create and insert claim
CClaimValue unused;
uint256 hash0(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); uint256 hash0(uint256S("0000000000000000000000000000000000000000000000000000000000000001"));
CMutableTransaction tx1 = BuildTransaction(hash0); CMutableTransaction tx1 = BuildTransaction(hash0);
uint160 claimId = ClaimIdHash(tx1.GetHash(), 0); uint160 claimId = ClaimIdHash(tx1.GetHash(), 0);
@ -304,7 +303,9 @@ BOOST_AUTO_TEST_CASE(iteratetrie_test)
BOOST_CHECK_EQUAL(node.children.size(), 1U); BOOST_CHECK_EQUAL(node.children.size(), 1U);
BOOST_CHECK(pclaimTrie->find("test", node)); BOOST_CHECK(pclaimTrie->find("test", node));
BOOST_CHECK_EQUAL(node.children.size(), 0U); BOOST_CHECK_EQUAL(node.children.size(), 0U);
BOOST_CHECK_EQUAL(node.data.claims.size(), 1); CClaimTrieData data;
BOOST_CHECK(pclaimTrie->find("test", data));
BOOST_CHECK_EQUAL(data.claims.size(), 1);
} }
BOOST_AUTO_TEST_CASE(trie_stays_consistent_test) BOOST_AUTO_TEST_CASE(trie_stays_consistent_test)