LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

flush ncc trie to disk at the same time pcoins and pblocktree are flushed to disk, so that if the program crashes they are at the same block

Browse source
This commit is contained in:
Jimmy Kiselak 2015-03-23 14:51:29 -04:00
parent 7813f3157b
commit c9785bb21f
5 changed files with 62 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/init.cpp
View file

@ -1046,7 +1046,7 @@ bool AppInit2(boost::thread_group& threadGroup)
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
pcoinsTip = new CCoinsViewCache(pcoinscatcher);
pnccTrie = new CNCCTrie();
pnccTrie = new CNCCTrie(false, fReindex);
if (fReindex)
pblocktree->WriteReindexing(true);

2
src/main.cpp
View file

@ -2081,6 +2081,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
}
}
// Finally flush the chainstate (which may refer to block index entries).
if (!pnccTrie->WriteToDisk())
return state.Abort("Failed to write to ncc trie database");
if (!pcoinsTip->Flush())
return state.Abort("Failed to write to coin database");
// Update best block in wallet (so we can detect restored wallets).

103
src/ncctrie.cpp
View file

@ -40,7 +40,6 @@ bool CNCCTrieNode::removeValue(uint256& txhash, uint32_t nOut, CNodeValue& val,
CNodeValue currentTop = values.front();
//std::vector<CNodeValue>::iterator position = std::find(values.begin(), values.end(), val);
std::vector<CNodeValue>::iterator position;
for (position = values.begin(); position != values.end(); ++position)
{
@ -214,11 +213,13 @@ bool CNCCTrie::recursiveCheckConsistency(CNCCTrieNode* node)
return calculatedHash == node->hash;
}
valueQueueType::iterator CNCCTrie::getQueueRow(int nHeight)
valueQueueType::iterator CNCCTrie::getQueueRow(int nHeight, bool createIfNotExists)
{
valueQueueType::iterator itQueueRow = valueQueue.find(nHeight);
if (itQueueRow == valueQueue.end())
{
if (!createIfNotExists)
return itQueueRow;
std::vector<CValueQueueEntry> queueRow;
std::pair<valueQueueType::iterator, bool> ret;
ret = valueQueue.insert(std::pair<int, std::vector<CValueQueueEntry> >(nHeight, queueRow));
@ -250,56 +251,54 @@ bool CNCCTrie::update(nodeCacheType& cache, hashMapType& hashes, const uint256&
// This can probably be optimized by checking each substring against
// the caches each time, but that will come after this is shown to
// work correctly.
// As far as saving to disk goes, the idea is to use the list of
// hashes to construct a list of (pointers to) nodes that have been
// altered in the update, and to construct a list of names of nodes
// that have been deleted, and to use a leveldb batch to write them
// all to disk. As of right now, txundo stuff will be handled by
// appending extra data to the normal txundo, which will call the
// normal insert/remove names, but obviously the opposite and in
// reverse order (though the order shouldn't ever matter).
// Disk strategy: keep a map of <string: dirty node>, where
// any nodes that are changed get put into the map, and any nodes
// to be deleted will simply be empty (no value, no children). Nodes
// whose hashes change will also be inserted into the map.
// As far as the queue goes, just keep a list of dirty queue entries.
// When the time comes, send all of that to disk in one batch, and
// empty the map/list.
bool success = true;
std::vector<std::string> deletedNames;
nodeCacheType changedNodes;
for (nodeCacheType::iterator itcache = cache.begin(); itcache != cache.end(); ++itcache)
{
CNCCTrieNode* pNode;
success = updateName(itcache->first, itcache->second, deletedNames, &pNode);
success = updateName(itcache->first, itcache->second);
if (!success)
return false;
changedNodes[itcache->first] = pNode;
}
for (hashMapType::iterator ithash = hashes.begin(); ithash != hashes.end(); ++ithash)
{
CNCCTrieNode* pNode;
success = updateHash(ithash->first, ithash->second, &pNode);
success = updateHash(ithash->first, ithash->second);
if (!success)
return false;
changedNodes[ithash->first] = pNode;
}
std::vector<int> vChangedQueueRows;
std::vector<int> vDeletedQueueRows;
for (valueQueueType::iterator itQueueCacheRow = queueCache.begin(); itQueueCacheRow != queueCache.end(); ++itQueueCacheRow)
{
if (itQueueCacheRow->second.empty())
{
vDeletedQueueRows.push_back(itQueueCacheRow->first);
deleteQueueRow(itQueueCacheRow->first);
}
else
{
vChangedQueueRows.push_back(itQueueCacheRow->first);
valueQueueType::iterator itQueueRow = getQueueRow(itQueueCacheRow->first);
valueQueueType::iterator itQueueRow = getQueueRow(itQueueCacheRow->first, true);
itQueueRow->second.swap(itQueueCacheRow->second);
}
vDirtyQueueRows.push_back(itQueueCacheRow->first);
}
BatchWrite(changedNodes, deletedNames, hashBlockIn, vChangedQueueRows, vDeletedQueueRows, nNewHeight);
hashBlock = hashBlockIn;
nCurrentHeight = nNewHeight;
return true;
}
bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames, CNCCTrieNode** pNodeRet)
void CNCCTrie::markNodeDirty(const std::string &name, CNCCTrieNode* node)
{
std::pair<nodeCacheType::iterator, bool> ret;
ret = dirtyNodes.insert(std::pair<std::string, CNCCTrieNode*>(name, node));
if (ret.second == false)
ret.first->second = node;
}
bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode)
{
CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
@ -323,7 +322,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
}
assert(current != NULL);
current->values.swap(updatedNode->values);
*pNodeRet = current;
markNodeDirty(name, current);
for (nodeMapType::iterator itchild = current->children.begin(); itchild != current->children.end();)
{
nodeMapType::iterator itupdatechild = updatedNode->children.find(itchild->first);
@ -334,7 +333,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
std::stringstream ss;
ss << name << itchild->first;
std::string newName = ss.str();
if (!recursiveNullify(itchild->second, newName, deletedNames))
if (!recursiveNullify(itchild->second, newName))
return false;
current->children.erase(itchild++);
}
@ -344,7 +343,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
return true;
}
bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames)
bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name)
{
assert(node != NULL);
for (nodeMapType::iterator itchild = node->children.begin(); itchild != node->children.end(); ++itchild)
@ -352,16 +351,16 @@ bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name, std::vect
std::stringstream ss;
ss << name << itchild->first;
std::string newName = ss.str();
if (!recursiveNullify(itchild->second, newName, deletedNames))
if (!recursiveNullify(itchild->second, newName))
return false;
}
node->children.clear();
markNodeDirty(name, NULL);
delete node;
deletedNames.push_back(name);
return true;
}
bool CNCCTrie::updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet)
bool CNCCTrie::updateHash(const std::string& name, uint256& hash)
{
CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
@ -373,45 +372,39 @@ bool CNCCTrie::updateHash(const std::string& name, uint256& hash, CNCCTrieNode**
}
assert(current != NULL);
current->hash = hash;
*pNodeRet = current;
markNodeDirty(name, current);
return true;
}
void CNCCTrie::BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const
{
LogPrintf("%s: Writing %s to disk with %d values\n", __func__, name, pNode->values.size());
batch.Write(std::make_pair('n', name), *pNode);
}
void CNCCTrie::BatchEraseNode(CLevelDBBatch& batch, const std::string& name) const
{
batch.Erase(std::make_pair('n', name));
if (pNode)
batch.Write(std::make_pair('n', name), *pNode);
else
batch.Erase(std::make_pair('n', name));
}
void CNCCTrie::BatchWriteQueueRow(CLevelDBBatch& batch, int nRowNum)
{
valueQueueType::iterator itQueueRow = getQueueRow(nRowNum);
batch.Write(std::make_pair('r', nRowNum), itQueueRow->second);
valueQueueType::iterator itQueueRow = getQueueRow(nRowNum, false);
if (itQueueRow != valueQueue.end())
batch.Write(std::make_pair('r', nRowNum), itQueueRow->second);
else
batch.Erase(std::make_pair('r', nRowNum));
}
void CNCCTrie::BatchEraseQueueRow(CLevelDBBatch& batch, int nRowNum)
{
batch.Erase(std::make_pair('r', nRowNum));
}
bool CNCCTrie::BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames, const uint256& hashBlockIn, std::vector<int> vChangedQueueRows, std::vector<int> vDeletedQueueRows, int nNewHeight)
bool CNCCTrie::WriteToDisk()
{
CLevelDBBatch batch;
for (nodeCacheType::iterator itcache = changedNodes.begin(); itcache != changedNodes.end(); ++itcache)
for (nodeCacheType::iterator itcache = dirtyNodes.begin(); itcache != dirtyNodes.end(); ++itcache)
BatchWriteNode(batch, itcache->first, itcache->second);
for (std::vector<std::string>::iterator itname = deletedNames.begin(); itname != deletedNames.end(); ++itname)
BatchEraseNode(batch, *itname);
for (std::vector<int>::iterator itRowNum = vChangedQueueRows.begin(); itRowNum != vChangedQueueRows.end(); ++itRowNum)
dirtyNodes.clear();
for (std::vector<int>::iterator itRowNum = vDirtyQueueRows.begin(); itRowNum != vDirtyQueueRows.end(); ++itRowNum)
BatchWriteQueueRow(batch, *itRowNum);
for (std::vector<int>::iterator itRowNum = vDeletedQueueRows.begin(); itRowNum != vDeletedQueueRows.end(); ++itRowNum)
BatchEraseQueueRow(batch, *itRowNum);
batch.Write('h', hashBlockIn);
batch.Write('t', nNewHeight);
vDirtyQueueRows.clear();
batch.Write('h', hashBlock);
batch.Write('t', nCurrentHeight);
return db.WriteBatch(batch);
}
@ -468,7 +461,7 @@ bool CNCCTrie::ReadFromDisk(bool check)
int nHeight;
ssKey >> nHeight;
CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
valueQueueType::iterator itQueueRow = getQueueRow(nHeight);
valueQueueType::iterator itQueueRow = getQueueRow(nHeight, true);
ssValue >> itQueueRow->second;
}
pcursor->Next();

17
src/ncctrie.h
View file

@ -26,7 +26,6 @@ public:
int nHeight;
int nValidAtHeight;
CNodeValue() {};
//CNodeValue(uint256 txhash, uint32_t nOut) : txhash(txhash), nOut(nOut), nAmount(0), nHeight(0), nValidAtHeight(0) {}
CNodeValue(uint256 txhash, uint32_t nOut, CAmount nAmount, int nHeight, int nValidAtHeight) : txhash(txhash), nOut(nOut), nAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight) {}
std::string ToString();
@ -149,11 +148,12 @@ class CNCCTrieCache;
class CNCCTrie
{
public:
CNCCTrie() : db(GetDataDir() / "ncctrie", 100, false, false), nCurrentHeight(1), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {}
CNCCTrie(bool fMemory = false, bool fWipe = false) : db(GetDataDir() / "ncctrie", 100, fMemory, fWipe), nCurrentHeight(1), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {}
uint256 getMerkleHash();
CLevelDBWrapper db;
bool empty() const;
bool checkConsistency();
bool WriteToDisk();
bool ReadFromDisk(bool check = false);
json_spirit::Array dumpToJSON() const;
bool getInfoForName(const std::string& name, CNodeValue& val) const;
@ -163,17 +163,20 @@ public:
friend class CNCCTrieCache;
private:
bool update(nodeCacheType& cache, hashMapType& hashes, const uint256& hashBlock, valueQueueType& queueCache, int nNewHeight);
bool updateName(const std::string& name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames, CNCCTrieNode** pNodeRet);
bool updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet);
bool recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames);
bool updateName(const std::string& name, CNCCTrieNode* updatedNode);
bool updateHash(const std::string& name, uint256& hash);
bool recursiveNullify(CNCCTrieNode* node, std::string& name);
bool recursiveCheckConsistency(CNCCTrieNode* node);
bool BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames, const uint256& hashBlock, std::vector<int> vChangedQueueRows, std::vector<int> vDeletedQueueRows, int nNewHeight);
bool InsertFromDisk(const std::string& name, CNCCTrieNode* node);
bool recursiveDumpToJSON(const std::string& name, const CNCCTrieNode* current, json_spirit::Array& ret) const;
CNCCTrieNode root;
uint256 hashBlock;
valueQueueType valueQueue;
valueQueueType::iterator getQueueRow(int nHeight);
valueQueueType::iterator getQueueRow(int nHeight, bool deleteIfNotExists);
nodeCacheType dirtyNodes;
std::vector<int> vDirtyQueueRows;
void markNodeDirty(const std::string& name, CNCCTrieNode* node);
void deleteQueueRow(int nHeight);
void BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const;
void BatchEraseNode(CLevelDBBatch& batch, const std::string& nome) const;

2
src/test/test_bitcoin.cpp
View file

@ -42,7 +42,7 @@ struct TestingSetup {
pblocktree = new CBlockTreeDB(1 << 20, true);
pcoinsdbview = new CCoinsViewDB(1 << 23, true);
pcoinsTip = new CCoinsViewCache(pcoinsdbview);
pnccTrie = new CNCCTrie();
pnccTrie = new CNCCTrie(true);
InitBlockIndex();
#ifdef ENABLE_WALLET
bool fFirstRun;