LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Add progress logging to claimtrie loading and validation. #408

Open
pseudoscalar wants to merge 1 commit from pseudoscalar/issue/363 into master
pull from: pseudoscalar/issue/363
merge into: LBRYCommunity:master
Conversation 2 Commits 1 Files changed 2 +51 -4
2 changed files with 51 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
src/claimtrie.cpp
View file

@ -442,8 +442,17 @@ uint256 recursiveMerkleHash(TIterator& it, const iCbType<TIterator>& process)
bool CClaimTrieCacheBase::recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const bool CClaimTrieCacheBase::recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const
{ {
struct CRecursiveBreak {}; struct CRecursiveBreak {};
int nodesVisited = 0;
int reportDone = 0;
using iterator = CClaimTrie::const_iterator; using iterator = CClaimTrie::const_iterator;
iCbType<iterator> process = [&failed, &process](iterator& it) { iCbType<iterator> process = [this, &reportDone, &nodesVisited, &failed, &process](iterator& it) {
int percentageDone = std::max(1, std::min(99, (int)((double)nodesVisited / (double)base->height() * 100)));
if (reportDone < percentageDone/10) {
// report every 10% step
LogPrintf("[%d%%]...", percentageDone); /* Continued */
reportDone = percentageDone/10;
}
nodesVisited++;
if (it->hash.IsNull() || it->hash != recursiveMerkleHash(it, process)) { if (it->hash.IsNull() || it->hash != recursiveMerkleHash(it, process)) {
failed = it.key(); failed = it.key();
throw CRecursiveBreak(); throw CRecursiveBreak();
@ -451,6 +460,7 @@ bool CClaimTrieCacheBase::recursiveCheckConsistency(CClaimTrie::const_iterator&
}; };
try { try {
LogPrintf("[0%%]..."); /* Continued */
process(it); process(it);
} catch (const CRecursiveBreak&) { } catch (const CRecursiveBreak&) {
return false; return false;
@ -576,11 +586,28 @@ bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
base->clear(); base->clear();
boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator()); boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
LogPrintf("[0%%]..."); /* Continued */
int totalRecords = 0;
for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) { for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
totalRecords++;
}
int reportDone = 0;
int recordsDone = 0;
int totalNodes = 0;
for (pcursor->SeekToFirst(); pcursor->Valid(); recordsDone++, pcursor->Next()) {
int percentageDone = std::max(1, std::min(99, (int)((double)recordsDone / (double)totalRecords * 50)));
if (reportDone < percentageDone/10) {
// report every 10% step
LogPrintf("[%d%%]...", percentageDone); /* Continued */
reportDone = percentageDone/10;
}
std::pair<uint8_t, std::string> key; std::pair<uint8_t, std::string> key;
if (!pcursor->GetKey(key) || key.first != TRIE_NODE) if (!pcursor->GetKey(key) || key.first != TRIE_NODE)
continue; continue;
totalNodes++;
CClaimTrieData data; CClaimTrieData data;
if (pcursor->GetValue(data)) { if (pcursor->GetValue(data)) {
if (data.empty()) { if (data.empty()) {
@ -599,7 +626,16 @@ bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
} }
} }
for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) { reportDone = 0;
recordsDone = 0;
LogPrintf("[50%%]..."); /* Continued */
for (pcursor->SeekToFirst(); pcursor->Valid(); recordsDone++, pcursor->Next()) {
int percentageDone = std::max(1, std::min(99, (int)((double)recordsDone / (double)totalRecords * 50)));
if (reportDone < percentageDone/10) {
// report every 10% step
LogPrintf("[%d%%]...", 50+percentageDone); /* Continued */
reportDone = percentageDone/10;
}
std::pair<uint8_t, std::string> key; std::pair<uint8_t, std::string> key;
if (!pcursor->GetKey(key) || key.first != TRIE_NODE) if (!pcursor->GetKey(key) || key.first != TRIE_NODE)
continue; continue;
@ -613,8 +649,9 @@ bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
base->db->Erase(key); // this uses a lot of memory and it's 1-time upgrade from 12.4 so we aren't going to batch it base->db->Erase(key); // this uses a lot of memory and it's 1-time upgrade from 12.4 so we aren't going to batch it
} }
} }
LogPrintf("[Done].\n");
LogPrintf("Checking claim trie consistency... "); LogPrintf("Checking claim trie consistency... \n");
if (checkConsistency()) { if (checkConsistency()) {
LogPrintf("consistent\n"); LogPrintf("consistent\n");
if (tip && tip->hashClaimTrie != getMerkleHash()) if (tip && tip->hashClaimTrie != getMerkleHash())

12
src/claimtrieforks.cpp
View file

@ -312,8 +312,17 @@ bool CClaimTrieCacheHashFork::recursiveCheckConsistency(CClaimTrie::const_iterat
return CClaimTrieCacheNormalizationFork::recursiveCheckConsistency(it, failed); return CClaimTrieCacheNormalizationFork::recursiveCheckConsistency(it, failed);
struct CRecursiveBreak {}; struct CRecursiveBreak {};
int nodesVisited = 0;
int reportDone = 0;
using iterator = CClaimTrie::const_iterator; using iterator = CClaimTrie::const_iterator;
iCbType<iterator> process = [&failed, &process](iterator& it) -> uint256 { iCbType<iterator> process = [this, &reportDone, &nodesVisited, &failed, &process](iterator& it) -> uint256 {
int percentageDone = std::max(1, std::min(99, (int)((double)nodesVisited / (double)base->height() * 100)));
if (reportDone < percentageDone/10) {
// report every 10% step
LogPrintf("[%d%%]...", percentageDone); /* Continued */
reportDone = percentageDone/10;
}
nodesVisited++;
if (it->hash.IsNull() || it->hash != recursiveBinaryTreeHash(it, process)) { if (it->hash.IsNull() || it->hash != recursiveBinaryTreeHash(it, process)) {
failed = it.key(); failed = it.key();
throw CRecursiveBreak(); throw CRecursiveBreak();
@ -322,6 +331,7 @@ bool CClaimTrieCacheHashFork::recursiveCheckConsistency(CClaimTrie::const_iterat
}; };
try { try {
LogPrintf("[0%%]..."); /* Continued */
process(it); process(it);
} catch (const CRecursiveBreak&) { } catch (const CRecursiveBreak&) {
return false; return false;