LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Trying to minimize disk reads / writes

Browse source 
Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>
This commit is contained in:
Anthony Fieroni 2019-08-28 19:13:50 +03:00 committed by Brannon King
parent d96253cd40
commit e9d37a8c81
3 changed files with 213 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

252
src/claimtrie.cpp
View file

@ -61,7 +61,7 @@ template <typename T, typename C>
auto findOutPoint(T& cont, const C& point) -> decltype(cont.begin()) auto findOutPoint(T& cont, const C& point) -> decltype(cont.begin())
{ {
using type = typename T::value_type; using type = typename T::value_type;
static_assert(std::is_same<typename std::remove_const<T>::type, std::vector<type>>::value, "T should be a vector type"); static_assert(std::is_same<typename std::decay<T>::type, std::vector<type>>::value, "T should be a vector type");
return std::find_if(cont.begin(), cont.end(), [&point](const type& val) { return std::find_if(cont.begin(), cont.end(), [&point](const type& val) {
return equals(val, point); return equals(val, point);
}); });
@ -135,19 +135,31 @@ bool CClaimTrie::SyncToDisk()
return db && db->Sync(); return db && db->Sync();
} }
template <typename Key, typename Container> template <typename T>
typename Container::value_type* getQueue(CDBWrapper& db, uint8_t dbkey, const Key& key, Container& queue, bool create) using rm_ref = typename std::remove_reference<T>::type;
template <typename Key, typename Map>
auto getRow(const CDBWrapper& db, uint8_t dbkey, const Key& key, Map& queue) -> COptional<rm_ref<decltype(queue.at(key))>>
{ {
auto itQueue = queue.find(key); auto it = queue.find(key);
if (itQueue != queue.end()) if (it != queue.end())
return &(*itQueue); return {&(it->second)};
typename Container::mapped_type row; typename Map::mapped_type row;
if (db.Read(std::make_pair(dbkey, key), row) || create) { if (db.Read(std::make_pair(dbkey, key), row))
auto ret = queue.insert(std::make_pair(key, row)); return {std::move(row)};
return {};
}
template <typename Key, typename Value>
Value* getQueue(const CDBWrapper& db, uint8_t dbkey, const Key& key, std::map<Key, Value>& queue, bool create)
{
auto row = getRow(db, dbkey, key, queue);
if (row.unique() || (!row && create)) {
auto ret = queue.emplace(key, row ? std::move(*row) : Value{});
assert(ret.second); assert(ret.second);
return &(*ret.first); return &(ret.first->second);
} }
return nullptr; return row;
} }
template <typename T> template <typename T>
@ -158,57 +170,95 @@ inline constexpr bool supportedType()
} }
template <> template <>
std::pair<const int, std::vector<queueEntryType<CClaimValue>>>* CClaimTrieCacheBase::getQueueCacheRow(int nHeight, bool createIfNotExists) std::vector<queueEntryType<CClaimValue>>* CClaimTrieCacheBase::getQueueCacheRow(int nHeight, bool createIfNotExists)
{ {
return getQueue(*(base->db), CLAIM_QUEUE_ROW, nHeight, claimQueueCache, createIfNotExists); return getQueue(*(base->db), CLAIM_QUEUE_ROW, nHeight, claimQueueCache, createIfNotExists);
} }
template <> template <>
std::pair<const int, std::vector<queueEntryType<CSupportValue>>>* CClaimTrieCacheBase::getQueueCacheRow(int nHeight, bool createIfNotExists) std::vector<queueEntryType<CSupportValue>>* CClaimTrieCacheBase::getQueueCacheRow(int nHeight, bool createIfNotExists)
{ {
return getQueue(*(base->db), SUPPORT_QUEUE_ROW, nHeight, supportQueueCache, createIfNotExists); return getQueue(*(base->db), SUPPORT_QUEUE_ROW, nHeight, supportQueueCache, createIfNotExists);
} }
template <typename T> template <typename T>
std::pair<const int, std::vector<queueEntryType<T>>>* CClaimTrieCacheBase::getQueueCacheRow(int, bool) std::vector<queueEntryType<T>>* CClaimTrieCacheBase::getQueueCacheRow(int, bool)
{ {
supportedType<T>(); supportedType<T>();
return nullptr; return nullptr;
} }
template <> template <>
typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow<CClaimValue>(const std::string& name, bool createIfNotExists) COptional<const std::vector<queueEntryType<CClaimValue>>> CClaimTrieCacheBase::getQueueCacheRow(int nHeight) const
{
return getRow(*(base->db), CLAIM_QUEUE_ROW, nHeight, claimQueueCache);
}
template <>
COptional<const std::vector<queueEntryType<CSupportValue>>> CClaimTrieCacheBase::getQueueCacheRow(int nHeight) const
{
return getRow(*(base->db), SUPPORT_QUEUE_ROW, nHeight, supportQueueCache);
}
template <typename T>
COptional<const std::vector<queueEntryType<T>>> CClaimTrieCacheBase::getQueueCacheRow(int) const
{
supportedType<T>();
return {};
}
template <>
queueNameRowType* CClaimTrieCacheBase::getQueueCacheNameRow<CClaimValue>(const std::string& name, bool createIfNotExists)
{ {
return getQueue(*(base->db), CLAIM_QUEUE_NAME_ROW, name, claimQueueNameCache, createIfNotExists); return getQueue(*(base->db), CLAIM_QUEUE_NAME_ROW, name, claimQueueNameCache, createIfNotExists);
} }
template <> template <>
typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow<CSupportValue>(const std::string& name, bool createIfNotExists) queueNameRowType* CClaimTrieCacheBase::getQueueCacheNameRow<CSupportValue>(const std::string& name, bool createIfNotExists)
{ {
return getQueue(*(base->db), SUPPORT_QUEUE_NAME_ROW, name, supportQueueNameCache, createIfNotExists); return getQueue(*(base->db), SUPPORT_QUEUE_NAME_ROW, name, supportQueueNameCache, createIfNotExists);
} }
template <typename T> template <typename T>
typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow(const std::string&, bool) queueNameRowType* CClaimTrieCacheBase::getQueueCacheNameRow(const std::string&, bool)
{ {
supportedType<T>(); supportedType<T>();
return nullptr; return nullptr;
} }
template <> template <>
typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueueCacheRow<CClaimValue>(int nHeight, bool createIfNotExists) COptional<const queueNameRowType> CClaimTrieCacheBase::getQueueCacheNameRow<CClaimValue>(const std::string& name) const
{
return getRow(*(base->db), CLAIM_QUEUE_NAME_ROW, name, claimQueueNameCache);
}
template <>
COptional<const queueNameRowType> CClaimTrieCacheBase::getQueueCacheNameRow<CSupportValue>(const std::string& name) const
{
return getRow(*(base->db), SUPPORT_QUEUE_NAME_ROW, name, supportQueueNameCache);
}
template <typename T>
COptional<const queueNameRowType> CClaimTrieCacheBase::getQueueCacheNameRow(const std::string&) const
{
supportedType<T>();
return {};
}
template <>
expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow<CClaimValue>(int nHeight, bool createIfNotExists)
{ {
return getQueue(*(base->db), CLAIM_EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists); return getQueue(*(base->db), CLAIM_EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists);
} }
template <> template <>
typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueueCacheRow<CSupportValue>(int nHeight, bool createIfNotExists) expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow<CSupportValue>(int nHeight, bool createIfNotExists)
{ {
return getQueue(*(base->db), SUPPORT_EXP_QUEUE_ROW, nHeight, supportExpirationQueueCache, createIfNotExists); return getQueue(*(base->db), SUPPORT_EXP_QUEUE_ROW, nHeight, supportExpirationQueueCache, createIfNotExists);
} }
template <typename T> template <typename T>
typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueueCacheRow(int, bool) expirationQueueRowType* CClaimTrieCacheBase::getExpirationQueueCacheRow(int, bool)
{ {
supportedType<T>(); supportedType<T>();
return nullptr; return nullptr;
@ -244,16 +294,16 @@ supportEntryType CClaimTrieCacheBase::getSupportsForName(const std::string& name
} }
template <typename T> template <typename T>
bool CClaimTrieCacheBase::haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) bool CClaimTrieCacheBase::haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const
{ {
supportedType<T>(); supportedType<T>();
if (auto nameRow = getQueueCacheNameRow<T>(name)) { if (auto nameRow = getQueueCacheNameRow<T>(name)) {
auto itNameRow = findOutPoint(nameRow->second, outPoint); auto itNameRow = findOutPoint(*nameRow, outPoint);
if (itNameRow != nameRow->second.end()) { if (itNameRow != nameRow->end()) {
nValidAtHeight = itNameRow->nHeight; nValidAtHeight = itNameRow->nHeight;
if (auto row = getQueueCacheRow<T>(nValidAtHeight)) { if (auto row = getQueueCacheRow<T>(nValidAtHeight)) {
auto iRow = findOutPoint(row->second, CNameOutPointType{name, outPoint}); auto iRow = findOutPoint(*row, CNameOutPointType{name, outPoint});
if (iRow != row->second.end()) { if (iRow != row->end()) {
if (iRow->second.nValidAtHeight != nValidAtHeight) if (iRow->second.nValidAtHeight != nValidAtHeight)
LogPrintf("%s: An inconsistency was found in the support queue. Please report this to the developers:\nDifferent nValidAtHeight between named queue and height queue\n: name: %s, txid: %s, nOut: %d, nValidAtHeight in named queue: %d, nValidAtHeight in height queue: %d current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, iRow->second.nValidAtHeight, nNextHeight); LogPrintf("%s: An inconsistency was found in the support queue. Please report this to the developers:\nDifferent nValidAtHeight between named queue and height queue\n: name: %s, txid: %s, nOut: %d, nValidAtHeight in named queue: %d, nValidAtHeight in height queue: %d current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, iRow->second.nValidAtHeight, nNextHeight);
return true; return true;
@ -265,20 +315,19 @@ bool CClaimTrieCacheBase::haveInQueue(const std::string& name, const COutPoint&
return false; return false;
} }
bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const
{ {
return haveInQueue<CClaimValue>(name, outPoint, nValidAtHeight); return haveInQueue<CClaimValue>(name, outPoint, nValidAtHeight);
} }
bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const
{ {
return haveInQueue<CSupportValue>(name, outPoint, nValidAtHeight); return haveInQueue<CSupportValue>(name, outPoint, nValidAtHeight);
} }
void CClaimTrie::recurseNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<recurseNodesCB> function) const { void CClaimTrie::recurseNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<recurseNodesCB> function) const {
CClaimTrieData data; CClaimTrieData data;
if (!find(name, data)) find(name, data);
data = {};
data.hash = current.hash; data.hash = current.hash;
data.flags |= current.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN; data.flags |= current.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
@ -286,8 +335,9 @@ void CClaimTrie::recurseNodes(const std::string& name, const CClaimTrieDataNode&
for (auto& child: current.children) { for (auto& child: current.children) {
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find(name + child, node)) auto childName = name + child;
recurseNodes(name + child, node, function); if (find(childName, node))
recurseNodes(childName, node, function);
} }
} }
@ -295,8 +345,8 @@ std::size_t CClaimTrie::getTotalNamesInTrie() const
{ {
std::size_t count = 0; std::size_t count = 0;
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find("", node)) if (find({}, node))
recurseNodes("", node, [&count](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) { recurseNodes({}, node, [&count](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
count += !data.empty(); count += !data.empty();
}); });
return count; return count;
@ -306,8 +356,8 @@ std::size_t CClaimTrie::getTotalClaimsInTrie() const
{ {
std::size_t count = 0; std::size_t count = 0;
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find("", node)) if (find({}, node))
recurseNodes("", node, [&count] recurseNodes({}, node, [&count]
(const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) { (const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
count += data.claims.size(); count += data.claims.size();
}); });
@ -318,8 +368,8 @@ CAmount CClaimTrie::getTotalValueOfClaimsInTrie(bool fControllingOnly) const
{ {
CAmount value_in_subtrie = 0; CAmount value_in_subtrie = 0;
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (find("", node)) if (find({}, node))
recurseNodes("", node, [&value_in_subtrie, fControllingOnly] recurseNodes({}, node, [&value_in_subtrie, fControllingOnly]
(const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) { (const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
for (const auto &claim : data.claims) { for (const auto &claim : data.claims) {
value_in_subtrie += claim.nAmount; value_in_subtrie += claim.nAmount;
@ -398,7 +448,7 @@ using iCbType = std::function<void(T&)>;
bool CClaimTrie::checkConsistency(const uint256& rootHash) const bool CClaimTrie::checkConsistency(const uint256& rootHash) const
{ {
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (!find("", node) || node.hash != rootHash) { if (!find({}, node) || node.hash != rootHash) {
if (rootHash == one) if (rootHash == one)
return true; return true;
@ -406,7 +456,7 @@ bool CClaimTrie::checkConsistency(const uint256& rootHash) const
} }
bool success = true; bool success = true;
recurseNodes("", node, [&success, this](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) { recurseNodes({}, node, [&success, this](const std::string &name, const CClaimTrieData &data, const std::vector<std::string>& children) {
if (!success) return; if (!success) return;
std::vector<uint8_t> vchToHash; std::vector<uint8_t> vchToHash;
@ -438,9 +488,10 @@ std::vector<std::pair<std::string, CClaimTrieDataNode>> CClaimTrie::nodes(const
std::vector<std::pair<std::string, CClaimTrieDataNode>> ret; std::vector<std::pair<std::string, CClaimTrieDataNode>> ret;
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (!find("", node)) if (!find({}, node))
return ret; return ret;
ret.emplace_back("", node);
ret.emplace_back(std::string{}, node);
std::string partialKey = key; std::string partialKey = key;
@ -473,7 +524,7 @@ bool CClaimTrie::contains(const std::string &key) const {
} }
bool CClaimTrie::empty() const { bool CClaimTrie::empty() const {
return !contains(""); return !contains({});
} }
bool CClaimTrie::find(const std::string& key, CClaimTrieDataNode &node) const { bool CClaimTrie::find(const std::string& key, CClaimTrieDataNode &node) const {
@ -533,17 +584,8 @@ bool CClaimTrieCacheBase::flush()
auto rootHash = getMerkleHash(); auto rootHash = getMerkleHash();
std::set<std::string> forDeletion;
for (const auto& nodeName : nodesToDelete) {
// TODO: we don't need to deserialize all the nodes right here
// we could be smarter about this and fill in the whole list in removeClaimFromTrie
auto nodes = base->nodes(nodeName);
for (auto& node : nodes)
forDeletion.insert(node.first);
}
for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) { for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) {
bool removed = forDeletion.erase(it.key()); bool removed = forDeleteFromBase.erase(it.key());
if (it->flags & CClaimTrieDataFlags::HASH_DIRTY) { if (it->flags & CClaimTrieDataFlags::HASH_DIRTY) {
CClaimTrieDataNode node; CClaimTrieDataNode node;
node.hash = it->hash; node.hash = it->hash;
@ -557,7 +599,7 @@ bool CClaimTrieCacheBase::flush()
} }
} }
for (auto& name: forDeletion) { for (auto& name: forDeleteFromBase) {
batch.Erase(std::make_pair(TRIE_NODE_CHILDREN, name)); batch.Erase(std::make_pair(TRIE_NODE_CHILDREN, name));
batch.Erase(std::make_pair(TRIE_NODE_CLAIMS, name)); batch.Erase(std::make_pair(TRIE_NODE_CLAIMS, name));
} }
@ -652,19 +694,16 @@ uint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(CClaimPrefixTrie::iterat
vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end()); vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
} }
auto ret = Hash(vchToHash.begin(), vchToHash.end()); return it->hash = Hash(vchToHash.begin(), vchToHash.end());
it->hash = ret;
return ret;
} }
uint256 CClaimTrieCacheBase::getMerkleHash() uint256 CClaimTrieCacheBase::getMerkleHash()
{ {
auto it = nodesToAddOrUpdate.begin(); if (auto it = nodesToAddOrUpdate.begin())
if (it)
return recursiveComputeMerkleHash(it); return recursiveComputeMerkleHash(it);
if (nodesToDelete.empty() && nodesAlreadyCached.empty()) { if (nodesToDelete.empty() && nodesAlreadyCached.empty()) {
CClaimTrieDataNode node; CClaimTrieDataNode node;
if (base->find("", node)) if (base->find({}, node))
return node.hash; // it may be valuable to have base cache its current root hash return node.hash; // it may be valuable to have base cache its current root hash
} }
return one; // we have no data or we deleted everything return one; // we have no data or we deleted everything
@ -694,8 +733,7 @@ CClaimPrefixTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& nam
if (nodesAlreadyCached.insert(node.first).second) { if (nodesAlreadyCached.insert(node.first).second) {
// do not insert nodes that are already present // do not insert nodes that are already present
CClaimTrieData data; CClaimTrieData data;
if (!base->find(node.first, data)) base->find(node.first, data);
data = {};
data.hash = node.second.hash; data.hash = node.second.hash;
data.flags = node.second.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN; data.flags = node.second.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
nodesToAddOrUpdate.insert(node.first, data); nodesToAddOrUpdate.insert(node.first, data);
@ -707,7 +745,7 @@ CClaimPrefixTrie::iterator CClaimTrieCacheBase::cacheData(const std::string& nam
if (!base->find(childKey, childData)) if (!base->find(childKey, childData))
childData = {}; childData = {};
CClaimTrieDataNode childNode; CClaimTrieDataNode childNode;
if (base->find(childKey, childNode)) { // TODO: can we eliminate this expensive lookup? if (base->find(childKey, childNode)) {
childData.hash = childNode.hash; childData.hash = childNode.hash;
childData.flags = childNode.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN; childData.flags = childNode.children.empty() ? 0 : CClaimTrieDataFlags::POTENTIAL_CHILDREN;
} }
@ -790,11 +828,12 @@ bool CClaimTrieCacheBase::removeClaimFromTrie(const std::string& name, const COu
it->reorderClaims(supports); it->reorderClaims(supports);
} else { } else {
// in case we pull a child into our spot; we will then need their kids for hash // in case we pull a child into our spot; we will then need their kids for hash
bool hasChild = false; bool hasChild = it.hasChildren();
for (auto& child: it.children()) { for (auto& child: it.children())
hasChild = true;
cacheData(child.key(), false); cacheData(child.key(), false);
}
for (auto& node : nodesToAddOrUpdate.nodes(name))
forDeleteFromBase.emplace(node.key());
nodesToAddOrUpdate.erase(name); nodesToAddOrUpdate.erase(name);
nodesToDelete.insert(name); nodesToDelete.insert(name);
@ -842,11 +881,11 @@ bool CClaimTrieCacheBase::addToQueue(const std::string& name, const T& value)
supportedType<T>(); supportedType<T>();
const auto newName = adjustNameForValidHeight(name, value.nValidAtHeight); const auto newName = adjustNameForValidHeight(name, value.nValidAtHeight);
auto itQueueCache = getQueueCacheRow<T>(value.nValidAtHeight, true); auto itQueueCache = getQueueCacheRow<T>(value.nValidAtHeight, true);
itQueueCache->second.emplace_back(newName, value); itQueueCache->emplace_back(newName, value);
auto itQueueName = getQueueCacheNameRow<T>(newName, true); auto itQueueName = getQueueCacheNameRow<T>(newName, true);
itQueueName->second.emplace_back(value.outPoint, value.nValidAtHeight); itQueueName->emplace_back(value.outPoint, value.nValidAtHeight);
auto itQueueExpiration = getExpirationQueueCacheRow<T>(value.nHeight + expirationTime(), true); auto itQueueExpiration = getExpirationQueueCacheRow<T>(value.nHeight + expirationTime(), true);
itQueueExpiration->second.emplace_back(newName, value.outPoint); itQueueExpiration->emplace_back(newName, value.outPoint);
return true; return true;
} }
@ -875,7 +914,7 @@ bool CClaimTrieCacheBase::undoSpend(const std::string& name, const T& value, int
supportedType<T>(); supportedType<T>();
if (nValidAtHeight < nNextHeight) { if (nValidAtHeight < nNextHeight) {
auto itQueueExpiration = getExpirationQueueCacheRow<T>(value.nHeight + expirationTime(), true); auto itQueueExpiration = getExpirationQueueCacheRow<T>(value.nHeight + expirationTime(), true);
itQueueExpiration->second.emplace_back(adjustNameForValidHeight(name, nValidAtHeight), value.outPoint); itQueueExpiration->emplace_back(adjustNameForValidHeight(name, nValidAtHeight), value.outPoint);
return addToCache(name, value, false); return addToCache(name, value, false);
} }
return addToQueue(name, value); return addToQueue(name, value);
@ -900,15 +939,15 @@ template <typename T>
bool CClaimTrieCacheBase::removeFromQueue(const std::string& name, const COutPoint& outPoint, T& value) bool CClaimTrieCacheBase::removeFromQueue(const std::string& name, const COutPoint& outPoint, T& value)
{ {
supportedType<T>(); supportedType<T>();
if (auto itQueueNameRow = getQueueCacheNameRow<T>(name)) { if (auto itQueueNameRow = getQueueCacheNameRow<T>(name, false)) {
auto itQueueName = findOutPoint(itQueueNameRow->second, outPoint); auto itQueueName = findOutPoint(*itQueueNameRow, outPoint);
if (itQueueName != itQueueNameRow->second.end()) { if (itQueueName != itQueueNameRow->end()) {
if (auto itQueueRow = getQueueCacheRow<T>(itQueueName->nHeight)) { if (auto itQueueRow = getQueueCacheRow<T>(itQueueName->nHeight, false)) {
auto itQueue = findOutPoint(itQueueRow->second, CNameOutPointType{name, outPoint}); auto itQueue = findOutPoint(*itQueueRow, CNameOutPointType{name, outPoint});
if (itQueue != itQueueRow->second.end()) { if (itQueue != itQueueRow->end()) {
std::swap(value, itQueue->second); std::swap(value, itQueue->second);
itQueueNameRow->second.erase(itQueueName); itQueueNameRow->erase(itQueueName);
itQueueRow->second.erase(itQueue); itQueueRow->erase(itQueue);
return true; return true;
} }
} }
@ -970,8 +1009,8 @@ bool CClaimTrieCacheBase::remove(T& value, const std::string& name, const COutPo
if (removeFromQueue(adjusted, outPoint, value) || removeFromCache(name, outPoint, value, fCheckTakeover)) { if (removeFromQueue(adjusted, outPoint, value) || removeFromCache(name, outPoint, value, fCheckTakeover)) {
int expirationHeight = value.nHeight + expirationTime(); int expirationHeight = value.nHeight + expirationTime();
if (auto itQueueRow = getExpirationQueueCacheRow<T>(expirationHeight)) if (auto itQueueRow = getExpirationQueueCacheRow<T>(expirationHeight, false))
eraseOutPoint(itQueueRow->second, CNameOutPointType{adjusted, outPoint}); eraseOutPoint(*itQueueRow, CNameOutPointType{adjusted, outPoint});
nValidAtHeight = value.nValidAtHeight; nValidAtHeight = value.nValidAtHeight;
return true; return true;
} }
@ -1103,10 +1142,10 @@ template <typename T>
void CClaimTrieCacheBase::undoIncrement(insertUndoType& insertUndo, std::vector<queueEntryType<T>>& expireUndo, std::set<T>* deleted) void CClaimTrieCacheBase::undoIncrement(insertUndoType& insertUndo, std::vector<queueEntryType<T>>& expireUndo, std::set<T>* deleted)
{ {
supportedType<T>(); supportedType<T>();
if (auto itQueueRow = getQueueCacheRow<T>(nNextHeight)) { if (auto itQueueRow = getQueueCacheRow<T>(nNextHeight, false)) {
for (const auto& itEntry : itQueueRow->second) { for (const auto& itEntry : *itQueueRow) {
if (auto itQueueNameRow = getQueueCacheNameRow<T>(itEntry.first)) { if (auto itQueueNameRow = getQueueCacheNameRow<T>(itEntry.first, false)) {
auto& points = itQueueNameRow->second; auto& points = *itQueueNameRow;
auto itQueueName = std::find_if(points.begin(), points.end(), [&itEntry, this](const COutPointHeightType& point) { auto itQueueName = std::find_if(points.begin(), points.end(), [&itEntry, this](const COutPointHeightType& point) {
return point.outPoint == itEntry.second.outPoint && point.nHeight == nNextHeight; return point.outPoint == itEntry.second.outPoint && point.nHeight == nNextHeight;
}); });
@ -1115,7 +1154,7 @@ void CClaimTrieCacheBase::undoIncrement(insertUndoType& insertUndo, std::vector<
} else { } else {
LogPrintf("%s: An inconsistency was found in the queue. Please report this to the developers:\nFound in height queue but not in named queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, itEntry.first, itEntry.second.outPoint.hash.GetHex(), itEntry.second.outPoint.n, itEntry.second.nValidAtHeight, nNextHeight); LogPrintf("%s: An inconsistency was found in the queue. Please report this to the developers:\nFound in height queue but not in named queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, itEntry.first, itEntry.second.outPoint.hash.GetHex(), itEntry.second.outPoint.n, itEntry.second.nValidAtHeight, nNextHeight);
LogPrintf("Elements found for that name:\n"); LogPrintf("Elements found for that name:\n");
for (const auto& itQueueNameInner : itQueueNameRow->second) for (const auto& itQueueNameInner : points)
LogPrintf("\ttxid: %s, nOut: %d, nValidAtHeight: %d\n", itQueueNameInner.outPoint.hash.GetHex(), itQueueNameInner.outPoint.n, itQueueNameInner.nHeight); LogPrintf("\ttxid: %s, nOut: %d, nValidAtHeight: %d\n", itQueueNameInner.outPoint.hash.GetHex(), itQueueNameInner.outPoint.n, itQueueNameInner.nHeight);
assert(false); assert(false);
} }
@ -1126,17 +1165,17 @@ void CClaimTrieCacheBase::undoIncrement(insertUndoType& insertUndo, std::vector<
addToCache(itEntry.first, itEntry.second, true); addToCache(itEntry.first, itEntry.second, true);
insertUndo.emplace_back(itEntry.first, itEntry.second.outPoint, itEntry.second.nValidAtHeight); insertUndo.emplace_back(itEntry.first, itEntry.second.outPoint, itEntry.second.nValidAtHeight);
} }
itQueueRow->second.clear(); itQueueRow->clear();
} }
if (auto itExpirationRow = getExpirationQueueCacheRow<T>(nNextHeight)) { if (auto itExpirationRow = getExpirationQueueCacheRow<T>(nNextHeight, false)) {
for (const auto& itEntry : itExpirationRow->second) { for (const auto& itEntry : *itExpirationRow) {
T value; T value;
assert(removeFromCache(itEntry.name, itEntry.outPoint, value, true)); assert(removeFromCache(itEntry.name, itEntry.outPoint, value, true));
expireUndo.emplace_back(itEntry.name, value); expireUndo.emplace_back(itEntry.name, value);
addTo(deleted, value); addTo(deleted, value);
} }
itExpirationRow->second.clear(); itExpirationRow->clear();
} }
} }
@ -1144,12 +1183,12 @@ template <typename T>
void CClaimTrieCacheBase::undoIncrement(const std::string& name, insertUndoType& insertUndo, std::vector<queueEntryType<T>>& expireUndo) void CClaimTrieCacheBase::undoIncrement(const std::string& name, insertUndoType& insertUndo, std::vector<queueEntryType<T>>& expireUndo)
{ {
supportedType<T>(); supportedType<T>();
if (auto itQueueNameRow = getQueueCacheNameRow<T>(name)) { if (auto itQueueNameRow = getQueueCacheNameRow<T>(name, false)) {
for (const auto& itQueueName : itQueueNameRow->second) { for (const auto& itQueueName : *itQueueNameRow) {
bool found = false; bool found = false;
// Pull those claims out of the height-based queue // Pull those claims out of the height-based queue
if (auto itQueueRow = getQueueCacheRow<T>(itQueueName.nHeight)) { if (auto itQueueRow = getQueueCacheRow<T>(itQueueName.nHeight, false)) {
auto& points = itQueueRow->second; auto& points = *itQueueRow;
auto itQueue = std::find_if(points.begin(), points.end(), [&name, &itQueueName](const queueEntryType<T>& point) { auto itQueue = std::find_if(points.begin(), points.end(), [&name, &itQueueName](const queueEntryType<T>& point) {
return name == point.first && point.second.outPoint == itQueueName.outPoint && point.second.nValidAtHeight == itQueueName.nHeight; return name == point.first && point.second.outPoint == itQueueName.outPoint && point.second.nValidAtHeight == itQueueName.nHeight;
}); });
@ -1169,7 +1208,7 @@ void CClaimTrieCacheBase::undoIncrement(const std::string& name, insertUndoType&
assert(found); assert(found);
} }
// remove all claims from the queue for that name // remove all claims from the queue for that name
itQueueNameRow->second.clear(); itQueueNameRow->clear();
} }
} }
@ -1265,12 +1304,13 @@ void CClaimTrieCacheBase::undoDecrement(insertUndoType& insertUndo, std::vector<
{ {
supportedType<T>(); supportedType<T>();
if (!expireUndo.empty()) { if (!expireUndo.empty()) {
auto itExpireRow = getExpirationQueueCacheRow<T>(nNextHeight, true);
for (auto itExpireUndo = expireUndo.crbegin(); itExpireUndo != expireUndo.crend(); ++itExpireUndo) { for (auto itExpireUndo = expireUndo.crbegin(); itExpireUndo != expireUndo.crend(); ++itExpireUndo) {
addToCache(itExpireUndo->first, itExpireUndo->second, false); addToCache(itExpireUndo->first, itExpireUndo->second, false);
addToIndex(index, itExpireUndo->first, itExpireUndo->second); addToIndex(index, itExpireUndo->first, itExpireUndo->second);
if (nNextHeight == itExpireUndo->second.nHeight + expirationTime()) if (nNextHeight == itExpireUndo->second.nHeight + expirationTime()) {
itExpireRow->second.emplace_back(itExpireUndo->first, itExpireUndo->second.outPoint); auto itExpireRow = getExpirationQueueCacheRow<T>(nNextHeight, true);
itExpireRow->emplace_back(itExpireUndo->first, itExpireUndo->second.outPoint);
}
} }
} }
@ -1283,8 +1323,8 @@ void CClaimTrieCacheBase::undoDecrement(insertUndoType& insertUndo, std::vector<
value.nValidAtHeight = itInsertUndo->nHeight; value.nValidAtHeight = itInsertUndo->nHeight;
auto itQueueRow = getQueueCacheRow<T>(itInsertUndo->nHeight, true); auto itQueueRow = getQueueCacheRow<T>(itInsertUndo->nHeight, true);
auto itQueueNameRow = getQueueCacheNameRow<T>(itInsertUndo->name, true); auto itQueueNameRow = getQueueCacheNameRow<T>(itInsertUndo->name, true);
itQueueRow->second.emplace_back(itInsertUndo->name, value); itQueueRow->emplace_back(itInsertUndo->name, value);
itQueueNameRow->second.emplace_back(itInsertUndo->outPoint, value.nValidAtHeight); itQueueNameRow->emplace_back(itInsertUndo->outPoint, value.nValidAtHeight);
} else { } else {
addTo(deleted, value); addTo(deleted, value);
} }
@ -1324,13 +1364,13 @@ void CClaimTrieCacheBase::reactivate(const expirationQueueRowType& row, int heig
supportedType<T>(); supportedType<T>();
for (auto& e: row) { for (auto& e: row) {
// remove and insert with new expiration time // remove and insert with new expiration time
if (auto itQueueRow = getExpirationQueueCacheRow<T>(height)) if (auto itQueueRow = getExpirationQueueCacheRow<T>(height, false))
eraseOutPoint(itQueueRow->second, CNameOutPointType{e.name, e.outPoint}); eraseOutPoint(*itQueueRow, CNameOutPointType{e.name, e.outPoint});
int extend_expiration = Params().GetConsensus().nExtendedClaimExpirationTime - Params().GetConsensus().nOriginalClaimExpirationTime; int extend_expiration = Params().GetConsensus().nExtendedClaimExpirationTime - Params().GetConsensus().nOriginalClaimExpirationTime;
int new_expiration_height = increment ? height + extend_expiration : height - extend_expiration; int new_expiration_height = increment ? height + extend_expiration : height - extend_expiration;
auto itQueueExpiration = getExpirationQueueCacheRow<T>(new_expiration_height, true); auto itQueueExpiration = getExpirationQueueCacheRow<T>(new_expiration_height, true);
itQueueExpiration->second.emplace_back(e.name, e.outPoint); itQueueExpiration->emplace_back(e.name, e.outPoint);
} }
} }
@ -1352,11 +1392,8 @@ int CClaimTrieCacheBase::getNumBlocksOfContinuousOwnership(const std::string& na
that->removalWorkaround.erase(hit); that->removalWorkaround.erase(hit);
return 0; return 0;
} }
auto it = nodesToAddOrUpdate.find(name); if (auto it = nodesToAddOrUpdate.find(name))
if (it && !it->empty()) return it->empty() ? 0 : nNextHeight - it->nHeightOfLastTakeover;
return nNextHeight - it->nHeightOfLastTakeover;
if (it) // we specifically ignore deleted nodes here to allow this to fall into the base lookup in that scenario
return 0;
CClaimTrieData data; CClaimTrieData data;
if (base->find(name, data) && !data.empty()) if (base->find(name, data) && !data.empty())
return nNextHeight - data.nHeightOfLastTakeover; return nNextHeight - data.nHeightOfLastTakeover;
@ -1387,6 +1424,7 @@ std::string CClaimTrieCacheBase::adjustNameForValidHeight(const std::string& nam
bool CClaimTrieCacheBase::clear() bool CClaimTrieCacheBase::clear()
{ {
forDeleteFromBase.clear();
nodesToAddOrUpdate.clear(); nodesToAddOrUpdate.clear();
claimsToAddToByIdIndex.clear(); claimsToAddToByIdIndex.clear();
supportCache.clear(); supportCache.clear();

69
src/claimtrie.h
View file

@ -398,6 +398,56 @@ struct CClaimTrieProof
int nHeightOfLastTakeover; int nHeightOfLastTakeover;
}; };
template <typename T>
class COptional
{
bool own;
T* value;
public:
COptional(T* value = nullptr) : own(false), value(value) {}
COptional(COptional&& o)
{
own = o.own;
value = o.value;
o.own = false;
o.value = nullptr;
}
COptional(T&& o) : own(true)
{
value = new T(std::move(o));
}
~COptional()
{
if (own)
delete value;
}
COptional& operator=(COptional&&) = delete;
bool unique() const
{
return own;
}
operator bool() const
{
return value;
}
operator T*() const
{
return value;
}
T* operator->() const
{
return value;
}
operator T&() const
{
return *value;
}
T& operator*() const
{
return *value;
}
};
template <typename T> template <typename T>
using queueEntryType = std::pair<std::string, T>; using queueEntryType = std::pair<std::string, T>;
@ -435,10 +485,10 @@ public:
bool ReadFromDisk(const CBlockIndex* tip); bool ReadFromDisk(const CBlockIndex* tip);
bool haveClaim(const std::string& name, const COutPoint& outPoint) const; bool haveClaim(const std::string& name, const COutPoint& outPoint) const;
bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight); bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
bool haveSupport(const std::string& name, const COutPoint& outPoint) const; bool haveSupport(const std::string& name, const COutPoint& outPoint) const;
bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight); bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight); bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
bool undoAddClaim(const std::string& name, const COutPoint& outPoint, int nHeight); bool undoAddClaim(const std::string& name, const COutPoint& outPoint, int nHeight);
@ -533,6 +583,7 @@ private:
std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
std::unordered_map<std::string, bool> takeoverWorkaround; std::unordered_map<std::string, bool> takeoverWorkaround;
std::unordered_set<std::string> removalWorkaround; std::unordered_set<std::string> removalWorkaround;
std::unordered_set<std::string> forDeleteFromBase;
bool shouldUseTakeoverWorkaround(const std::string& key) const; bool shouldUseTakeoverWorkaround(const std::string& key) const;
void addTakeoverWorkaroundPotential(const std::string& key); void addTakeoverWorkaroundPotential(const std::string& key);
@ -547,16 +598,22 @@ private:
bool validateTrieConsistency(const CBlockIndex* tip); bool validateTrieConsistency(const CBlockIndex* tip);
template <typename T> template <typename T>
std::pair<const int, std::vector<queueEntryType<T>>>* getQueueCacheRow(int nHeight, bool createIfNotExists = false); std::vector<queueEntryType<T>>* getQueueCacheRow(int nHeight, bool createIfNotExists);
template <typename T> template <typename T>
typename queueNameType::value_type* getQueueCacheNameRow(const std::string& name, bool createIfNotExists = false); COptional<const std::vector<queueEntryType<T>>> getQueueCacheRow(int nHeight) const;
template <typename T> template <typename T>
typename expirationQueueType::value_type* getExpirationQueueCacheRow(int nHeight, bool createIfNotExists = false); queueNameRowType* getQueueCacheNameRow(const std::string& name, bool createIfNotExists);
template <typename T> template <typename T>
bool haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight); COptional<const queueNameRowType> getQueueCacheNameRow(const std::string& name) const;
template <typename T>
expirationQueueRowType* getExpirationQueueCacheRow(int nHeight, bool createIfNotExists);
template <typename T>
bool haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
template <typename T> template <typename T>
T add(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight); T add(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);

6
src/prefixtrie.h
View file

@ -9,6 +9,10 @@
#include <type_traits> #include <type_traits>
#include <vector> #include <vector>
#include <boost/container/flat_map.hpp>
namespace bc = boost::container;
template <typename TKey, typename TData> template <typename TKey, typename TData>
class CPrefixTrie class CPrefixTrie
{ {
@ -17,7 +21,7 @@ class CPrefixTrie
template <bool> template <bool>
friend class Iterator; friend class Iterator;
friend class CPrefixTrie<TKey, TData>; friend class CPrefixTrie<TKey, TData>;
std::map<TKey, std::shared_ptr<Node>> children; bc::flat_map<TKey, std::shared_ptr<Node>> children;
public: public:
Node() = default; Node() = default;