LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

store ncc trie insertion queues on disk, so that eventually removal queues can be stored on disk the same way

Browse source
This commit is contained in:
Jimmy Kiselak 2015-05-21 13:50:40 -04:00
parent 6bca5ead23
commit 71a28f8d8f
2 changed files with 75 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

129
src/ncctrie.cpp
View file

@ -123,7 +123,34 @@ bool CNCCTrie::empty() const
bool CNCCTrie::queueEmpty() const
{
return valueQueue.empty();
for (valueQueueType::const_iterator itRow = dirtyQueueRows.begin(); itRow != dirtyQueueRows.end(); ++itRow)
{
if (!itRow->second.empty())
return false;
}
boost::scoped_ptr<leveldb::Iterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator());
pcursor->SeekToFirst();
while (pcursor->Valid())
{
try
{
leveldb::Slice slKey = pcursor->key();
CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION);
char chType;
ssKey >> chType;
if (chType == 'r')
{
return false;
}
}
catch (const std::exception& e)
{
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
}
pcursor->Next();
}
return true;
}
void CNCCTrie::clear()
@ -300,29 +327,29 @@ bool CNCCTrie::recursiveCheckConsistency(CNCCTrieNode* node)
return calculatedHash == node->hash;
}
valueQueueType::iterator CNCCTrie::getQueueRow(int nHeight, bool createIfNotExists)
bool CNCCTrie::getQueueRow(int nHeight, std::vector<CValueQueueEntry>& row)
{
valueQueueType::iterator itQueueRow = valueQueue.find(nHeight);
if (itQueueRow == valueQueue.end())
valueQueueType::iterator itQueueRow = dirtyQueueRows.find(nHeight);
if (itQueueRow != dirtyQueueRows.end())
{
if (!createIfNotExists)
return itQueueRow;
std::vector<CValueQueueEntry> queueRow;
row = itQueueRow->second;
return true;
}
return db.Read(std::make_pair('r', nHeight), row);
}
void CNCCTrie::updateQueueRow(int nHeight, std::vector<CValueQueueEntry>& row)
{
valueQueueType::iterator itQueueRow = dirtyQueueRows.find(nHeight);
if (itQueueRow == dirtyQueueRows.end())
{
std::vector<CValueQueueEntry> newRow;
std::pair<valueQueueType::iterator, bool> ret;
ret = valueQueue.insert(std::pair<int, std::vector<CValueQueueEntry> >(nHeight, queueRow));
ret = dirtyQueueRows.insert(std::pair<int, std::vector<CValueQueueEntry> >(nHeight, newRow));
assert(ret.second);
itQueueRow = ret.first;
}
return itQueueRow;
}
void CNCCTrie::deleteQueueRow(int nHeight)
{
valueQueueType::iterator itQueueRow = valueQueue.find(nHeight);
if (itQueueRow != valueQueue.end())
{
valueQueue.erase(itQueueRow);
}
itQueueRow->second.swap(row);
}
bool CNCCTrie::update(nodeCacheType& cache, hashMapType& hashes, const uint256& hashBlockIn, valueQueueType& queueCache, int nNewHeight)
@ -361,16 +388,7 @@ bool CNCCTrie::update(nodeCacheType& cache, hashMapType& hashes, const uint256&
}
for (valueQueueType::iterator itQueueCacheRow = queueCache.begin(); itQueueCacheRow != queueCache.end(); ++itQueueCacheRow)
{
if (itQueueCacheRow->second.empty())
{
deleteQueueRow(itQueueCacheRow->first);
}
else
{
valueQueueType::iterator itQueueRow = getQueueRow(itQueueCacheRow->first, true);
itQueueRow->second.swap(itQueueCacheRow->second);
}
vDirtyQueueRows.push_back(itQueueCacheRow->first);
updateQueueRow(itQueueCacheRow->first, itQueueCacheRow->second);
}
hashBlock = hashBlockIn;
nCurrentHeight = nNewHeight;
@ -472,13 +490,19 @@ void CNCCTrie::BatchWriteNode(CLevelDBBatch& batch, const std::string& name, con
batch.Erase(std::make_pair('n', name));
}
void CNCCTrie::BatchWriteQueueRow(CLevelDBBatch& batch, int nRowNum)
void CNCCTrie::BatchWriteQueueRows(CLevelDBBatch& batch)
{
valueQueueType::iterator itQueueRow = getQueueRow(nRowNum, false);
if (itQueueRow != valueQueue.end())
batch.Write(std::make_pair('r', nRowNum), itQueueRow->second);
else
batch.Erase(std::make_pair('r', nRowNum));
for (valueQueueType::iterator itQueue = dirtyQueueRows.begin(); itQueue != dirtyQueueRows.end(); ++itQueue)
{
if (itQueue->second.empty())
{
batch.Erase(std::make_pair('r', itQueue->first));
}
else
{
batch.Write(std::make_pair('r', itQueue->first), itQueue->second);
}
}
}
bool CNCCTrie::WriteToDisk()
@ -487,9 +511,8 @@ bool CNCCTrie::WriteToDisk()
for (nodeCacheType::iterator itcache = dirtyNodes.begin(); itcache != dirtyNodes.end(); ++itcache)
BatchWriteNode(batch, itcache->first, itcache->second);
dirtyNodes.clear();
for (std::vector<int>::iterator itRowNum = vDirtyQueueRows.begin(); itRowNum != vDirtyQueueRows.end(); ++itRowNum)
BatchWriteQueueRow(batch, *itRowNum);
vDirtyQueueRows.clear();
BatchWriteQueueRows(batch);
dirtyQueueRows.clear();
batch.Write('h', hashBlock);
batch.Write('t', nCurrentHeight);
return db.WriteBatch(batch);
@ -542,15 +565,6 @@ bool CNCCTrie::ReadFromDisk(bool check)
if (!InsertFromDisk(name, node))
return false;
}
else if (chType == 'r')
{
leveldb::Slice slValue = pcursor->value();
int nHeight;
ssKey >> nHeight;
CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
valueQueueType::iterator itQueueRow = getQueueRow(nHeight, true);
ssValue >> itQueueRow->second;
}
pcursor->Next();
}
catch (const std::exception& e)
@ -884,14 +898,10 @@ valueQueueType::iterator CNCCTrieCache::getQueueCacheRow(int nHeight, bool creat
// Have to make a new row it put in the cache, if createIfNotExists is true
std::vector<CValueQueueEntry> queueRow;
// If the row exists in the base, copy its values into the new row.
valueQueueType::iterator itBaseQueueRow = base->valueQueue.find(nHeight);
if (itBaseQueueRow == base->valueQueue.end())
{
bool exists = base->getQueueRow(nHeight, queueRow);
if (!exists)
if (!createIfNotExists)
return itQueueRow;
}
else
queueRow = itBaseQueueRow->second;
// Stick the new row in the cache
std::pair<valueQueueType::iterator, bool> ret;
ret = valueQueueCache.insert(std::pair<int, std::vector<CValueQueueEntry> >(nHeight, queueRow));
@ -1036,13 +1046,16 @@ bool CNCCTrieCache::decrementBlock(CNCCTrieQueueUndo& undo) const
{
LogPrintf("%s: nCurrentHeight (before decrement): %d\n", __func__, nCurrentHeight);
nCurrentHeight--;
valueQueueType::iterator itQueueRow = getQueueCacheRow(nCurrentHeight, true);
for (CNCCTrieQueueUndo::iterator itUndo = undo.begin(); itUndo != undo.end(); ++itUndo)
if (undo.begin() != undo.end())
{
int nValidHeightInTrie;
assert(removeClaimFromTrie(itUndo->name, itUndo->val.txhash, itUndo->val.nOut, nValidHeightInTrie));
assert(nValidHeightInTrie == itUndo->val.nValidAtHeight);
itQueueRow->second.push_back(*itUndo);
valueQueueType::iterator itQueueRow = getQueueCacheRow(nCurrentHeight, true);
for (CNCCTrieQueueUndo::iterator itUndo = undo.begin(); itUndo != undo.end(); ++itUndo)
{
int nValidHeightInTrie;
assert(removeClaimFromTrie(itUndo->name, itUndo->val.txhash, itUndo->val.nOut, nValidHeightInTrie));
assert(nValidHeightInTrie == itUndo->val.nValidAtHeight);
itQueueRow->second.push_back(*itUndo);
}
}
return true;
}

9
src/ncctrie.h
View file

@ -159,6 +159,7 @@ public:
bool getInfoForName(const std::string& name, CNodeValue& val) const;
int nCurrentHeight;
bool queueEmpty() const;
bool getQueueRow(int nHeight, std::vector<CValueQueueEntry>& row);
bool haveClaim(const std::string& name, const uint256& txhash, uint32_t nOut) const;
unsigned int getTotalNamesInTrie() const;
unsigned int getTotalClaimsInTrie() const;
@ -178,17 +179,15 @@ private:
bool recursiveDumpToJSON(const std::string& name, const CNCCTrieNode* current, json_spirit::Array& ret) const;
CNCCTrieNode root;
uint256 hashBlock;
valueQueueType valueQueue;
valueQueueType::iterator getQueueRow(int nHeight, bool deleteIfNotExists);
valueQueueType dirtyQueueRows;
nodeCacheType dirtyNodes;
std::vector<int> vDirtyQueueRows;
void markNodeDirty(const std::string& name, CNCCTrieNode* node);
void deleteQueueRow(int nHeight);
void updateQueueRow(int nHeight, std::vector<CValueQueueEntry>& row);
void BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const;
void BatchEraseNode(CLevelDBBatch& batch, const std::string& nome) const;
void BatchWriteQueueRow(CLevelDBBatch& batch, int nRowNum);
void BatchEraseQueueRow(CLevelDBBatch& batch, int nRowNum);
void BatchWriteQueueRows(CLevelDBBatch& batch);
};
class CNCCTrieCache