LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Rename CClaimTrieCache to CClaimTrieUpdateBuffer #156

Closed
kaykurokawa wants to merge 276 commits from cclaimtriecache_change into master
pull from: cclaimtriecache_change
merge into: LBRYCommunity:master
Conversation 3 Commits 276 Files changed 116 +62 -64
5 changed files with 62 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit c9785bb21f - Show all commits

2
src/init.cpp
View file

@ -1046,7 +1046,7 @@ bool AppInit2(boost::thread_group& threadGroup)
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
pcoinsTip = new CCoinsViewCache(pcoinscatcher); pcoinsTip = new CCoinsViewCache(pcoinscatcher);
pnccTrie = new CNCCTrie(); pnccTrie = new CNCCTrie(false, fReindex);
if (fReindex) if (fReindex)
pblocktree->WriteReindexing(true); pblocktree->WriteReindexing(true);

2
src/main.cpp
View file

@ -2081,6 +2081,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
} }
} }
// Finally flush the chainstate (which may refer to block index entries). // Finally flush the chainstate (which may refer to block index entries).
if (!pnccTrie->WriteToDisk())
return state.Abort("Failed to write to ncc trie database");
if (!pcoinsTip->Flush()) if (!pcoinsTip->Flush())
return state.Abort("Failed to write to coin database"); return state.Abort("Failed to write to coin database");
// Update best block in wallet (so we can detect restored wallets). // Update best block in wallet (so we can detect restored wallets).

95
src/ncctrie.cpp
View file

@ -40,7 +40,6 @@ bool CNCCTrieNode::removeValue(uint256& txhash, uint32_t nOut, CNodeValue& val,
CNodeValue currentTop = values.front(); CNodeValue currentTop = values.front();
//std::vector<CNodeValue>::iterator position = std::find(values.begin(), values.end(), val);
std::vector<CNodeValue>::iterator position; std::vector<CNodeValue>::iterator position;
for (position = values.begin(); position != values.end(); ++position) for (position = values.begin(); position != values.end(); ++position)
{ {
@ -214,11 +213,13 @@ bool CNCCTrie::recursiveCheckConsistency(CNCCTrieNode* node)
return calculatedHash == node->hash; return calculatedHash == node->hash;
} }
valueQueueType::iterator CNCCTrie::getQueueRow(int nHeight) valueQueueType::iterator CNCCTrie::getQueueRow(int nHeight, bool createIfNotExists)
{ {
valueQueueType::iterator itQueueRow = valueQueue.find(nHeight); valueQueueType::iterator itQueueRow = valueQueue.find(nHeight);
if (itQueueRow == valueQueue.end()) if (itQueueRow == valueQueue.end())
{ {
if (!createIfNotExists)
return itQueueRow;
std::vector<CValueQueueEntry> queueRow; std::vector<CValueQueueEntry> queueRow;
std::pair<valueQueueType::iterator, bool> ret; std::pair<valueQueueType::iterator, bool> ret;
ret = valueQueue.insert(std::pair<int, std::vector<CValueQueueEntry> >(nHeight, queueRow)); ret = valueQueue.insert(std::pair<int, std::vector<CValueQueueEntry> >(nHeight, queueRow));
@ -250,56 +251,54 @@ bool CNCCTrie::update(nodeCacheType& cache, hashMapType& hashes, const uint256&
// This can probably be optimized by checking each substring against // This can probably be optimized by checking each substring against
// the caches each time, but that will come after this is shown to // the caches each time, but that will come after this is shown to
// work correctly. // work correctly.
// As far as saving to disk goes, the idea is to use the list of // Disk strategy: keep a map of <string: dirty node>, where
// hashes to construct a list of (pointers to) nodes that have been // any nodes that are changed get put into the map, and any nodes
// altered in the update, and to construct a list of names of nodes // to be deleted will simply be empty (no value, no children). Nodes
// that have been deleted, and to use a leveldb batch to write them // whose hashes change will also be inserted into the map.
// all to disk. As of right now, txundo stuff will be handled by // As far as the queue goes, just keep a list of dirty queue entries.
// appending extra data to the normal txundo, which will call the // When the time comes, send all of that to disk in one batch, and
// normal insert/remove names, but obviously the opposite and in // empty the map/list.
// reverse order (though the order shouldn't ever matter).
bool success = true; bool success = true;
std::vector<std::string> deletedNames;
nodeCacheType changedNodes;
for (nodeCacheType::iterator itcache = cache.begin(); itcache != cache.end(); ++itcache) for (nodeCacheType::iterator itcache = cache.begin(); itcache != cache.end(); ++itcache)
{ {
CNCCTrieNode* pNode; success = updateName(itcache->first, itcache->second);
success = updateName(itcache->first, itcache->second, deletedNames, &pNode);
if (!success) if (!success)
return false; return false;
changedNodes[itcache->first] = pNode;
} }
for (hashMapType::iterator ithash = hashes.begin(); ithash != hashes.end(); ++ithash) for (hashMapType::iterator ithash = hashes.begin(); ithash != hashes.end(); ++ithash)
{ {
CNCCTrieNode* pNode; success = updateHash(ithash->first, ithash->second);
success = updateHash(ithash->first, ithash->second, &pNode);
if (!success) if (!success)
return false; return false;
changedNodes[ithash->first] = pNode;
} }
std::vector<int> vChangedQueueRows;
std::vector<int> vDeletedQueueRows;
for (valueQueueType::iterator itQueueCacheRow = queueCache.begin(); itQueueCacheRow != queueCache.end(); ++itQueueCacheRow) for (valueQueueType::iterator itQueueCacheRow = queueCache.begin(); itQueueCacheRow != queueCache.end(); ++itQueueCacheRow)
{ {
if (itQueueCacheRow->second.empty()) if (itQueueCacheRow->second.empty())
{ {
vDeletedQueueRows.push_back(itQueueCacheRow->first);
deleteQueueRow(itQueueCacheRow->first); deleteQueueRow(itQueueCacheRow->first);
} }
else else
{ {
vChangedQueueRows.push_back(itQueueCacheRow->first); valueQueueType::iterator itQueueRow = getQueueRow(itQueueCacheRow->first, true);
valueQueueType::iterator itQueueRow = getQueueRow(itQueueCacheRow->first);
itQueueRow->second.swap(itQueueCacheRow->second); itQueueRow->second.swap(itQueueCacheRow->second);
} }
vDirtyQueueRows.push_back(itQueueCacheRow->first);
} }
BatchWrite(changedNodes, deletedNames, hashBlockIn, vChangedQueueRows, vDeletedQueueRows, nNewHeight);
hashBlock = hashBlockIn; hashBlock = hashBlockIn;
nCurrentHeight = nNewHeight; nCurrentHeight = nNewHeight;
return true; return true;
} }
bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames, CNCCTrieNode** pNodeRet) void CNCCTrie::markNodeDirty(const std::string &name, CNCCTrieNode* node)
{
std::pair<nodeCacheType::iterator, bool> ret;
ret = dirtyNodes.insert(std::pair<std::string, CNCCTrieNode*>(name, node));
if (ret.second == false)
ret.first->second = node;
}
bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode)
{ {
CNCCTrieNode* current = &root; CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname) for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
@ -323,7 +322,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
} }
assert(current != NULL); assert(current != NULL);
current->values.swap(updatedNode->values); current->values.swap(updatedNode->values);
*pNodeRet = current; markNodeDirty(name, current);
for (nodeMapType::iterator itchild = current->children.begin(); itchild != current->children.end();) for (nodeMapType::iterator itchild = current->children.begin(); itchild != current->children.end();)
{ {
nodeMapType::iterator itupdatechild = updatedNode->children.find(itchild->first); nodeMapType::iterator itupdatechild = updatedNode->children.find(itchild->first);
@ -334,7 +333,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
std::stringstream ss; std::stringstream ss;
ss << name << itchild->first; ss << name << itchild->first;
std::string newName = ss.str(); std::string newName = ss.str();
if (!recursiveNullify(itchild->second, newName, deletedNames)) if (!recursiveNullify(itchild->second, newName))
return false; return false;
current->children.erase(itchild++); current->children.erase(itchild++);
} }
@ -344,7 +343,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
return true; return true;
} }
bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames) bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name)
{ {
assert(node != NULL); assert(node != NULL);
for (nodeMapType::iterator itchild = node->children.begin(); itchild != node->children.end(); ++itchild) for (nodeMapType::iterator itchild = node->children.begin(); itchild != node->children.end(); ++itchild)
@ -352,16 +351,16 @@ bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name, std::vect
std::stringstream ss; std::stringstream ss;
ss << name << itchild->first; ss << name << itchild->first;
std::string newName = ss.str(); std::string newName = ss.str();
if (!recursiveNullify(itchild->second, newName, deletedNames)) if (!recursiveNullify(itchild->second, newName))
return false; return false;
} }
node->children.clear(); node->children.clear();
markNodeDirty(name, NULL);
delete node; delete node;
deletedNames.push_back(name);
return true; return true;
} }
bool CNCCTrie::updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet) bool CNCCTrie::updateHash(const std::string& name, uint256& hash)
{ {
CNCCTrieNode* current = &root; CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname) for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
@ -373,45 +372,39 @@ bool CNCCTrie::updateHash(const std::string& name, uint256& hash, CNCCTrieNode**
} }
assert(current != NULL); assert(current != NULL);
current->hash = hash; current->hash = hash;
*pNodeRet = current; markNodeDirty(name, current);
return true; return true;
} }
void CNCCTrie::BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const void CNCCTrie::BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const
{ {
LogPrintf("%s: Writing %s to disk with %d values\n", __func__, name, pNode->values.size()); LogPrintf("%s: Writing %s to disk with %d values\n", __func__, name, pNode->values.size());
if (pNode)
batch.Write(std::make_pair('n', name), *pNode); batch.Write(std::make_pair('n', name), *pNode);
} else
void CNCCTrie::BatchEraseNode(CLevelDBBatch& batch, const std::string& name) const
{
batch.Erase(std::make_pair('n', name)); batch.Erase(std::make_pair('n', name));
} }
void CNCCTrie::BatchWriteQueueRow(CLevelDBBatch& batch, int nRowNum) void CNCCTrie::BatchWriteQueueRow(CLevelDBBatch& batch, int nRowNum)
{ {
valueQueueType::iterator itQueueRow = getQueueRow(nRowNum); valueQueueType::iterator itQueueRow = getQueueRow(nRowNum, false);
if (itQueueRow != valueQueue.end())
batch.Write(std::make_pair('r', nRowNum), itQueueRow->second); batch.Write(std::make_pair('r', nRowNum), itQueueRow->second);
} else
void CNCCTrie::BatchEraseQueueRow(CLevelDBBatch& batch, int nRowNum)
{
batch.Erase(std::make_pair('r', nRowNum)); batch.Erase(std::make_pair('r', nRowNum));
} }
bool CNCCTrie::BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames, const uint256& hashBlockIn, std::vector<int> vChangedQueueRows, std::vector<int> vDeletedQueueRows, int nNewHeight) bool CNCCTrie::WriteToDisk()
{ {
CLevelDBBatch batch; CLevelDBBatch batch;
for (nodeCacheType::iterator itcache = changedNodes.begin(); itcache != changedNodes.end(); ++itcache) for (nodeCacheType::iterator itcache = dirtyNodes.begin(); itcache != dirtyNodes.end(); ++itcache)
BatchWriteNode(batch, itcache->first, itcache->second); BatchWriteNode(batch, itcache->first, itcache->second);
for (std::vector<std::string>::iterator itname = deletedNames.begin(); itname != deletedNames.end(); ++itname) dirtyNodes.clear();
BatchEraseNode(batch, *itname); for (std::vector<int>::iterator itRowNum = vDirtyQueueRows.begin(); itRowNum != vDirtyQueueRows.end(); ++itRowNum)
for (std::vector<int>::iterator itRowNum = vChangedQueueRows.begin(); itRowNum != vChangedQueueRows.end(); ++itRowNum)
BatchWriteQueueRow(batch, *itRowNum); BatchWriteQueueRow(batch, *itRowNum);
for (std::vector<int>::iterator itRowNum = vDeletedQueueRows.begin(); itRowNum != vDeletedQueueRows.end(); ++itRowNum) vDirtyQueueRows.clear();
BatchEraseQueueRow(batch, *itRowNum); batch.Write('h', hashBlock);
batch.Write('h', hashBlockIn); batch.Write('t', nCurrentHeight);
batch.Write('t', nNewHeight);
return db.WriteBatch(batch); return db.WriteBatch(batch);
} }
@ -468,7 +461,7 @@ bool CNCCTrie::ReadFromDisk(bool check)
int nHeight; int nHeight;
ssKey >> nHeight; ssKey >> nHeight;
CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
valueQueueType::iterator itQueueRow = getQueueRow(nHeight); valueQueueType::iterator itQueueRow = getQueueRow(nHeight, true);
ssValue >> itQueueRow->second; ssValue >> itQueueRow->second;
} }
pcursor->Next(); pcursor->Next();

17
src/ncctrie.h
View file

@ -26,7 +26,6 @@ public:
int nHeight; int nHeight;
int nValidAtHeight; int nValidAtHeight;
CNodeValue() {}; CNodeValue() {};
//CNodeValue(uint256 txhash, uint32_t nOut) : txhash(txhash), nOut(nOut), nAmount(0), nHeight(0), nValidAtHeight(0) {}
CNodeValue(uint256 txhash, uint32_t nOut, CAmount nAmount, int nHeight, int nValidAtHeight) : txhash(txhash), nOut(nOut), nAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight) {} CNodeValue(uint256 txhash, uint32_t nOut, CAmount nAmount, int nHeight, int nValidAtHeight) : txhash(txhash), nOut(nOut), nAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight) {}
std::string ToString(); std::string ToString();
@ -149,11 +148,12 @@ class CNCCTrieCache;
class CNCCTrie class CNCCTrie
{ {
public: public:
CNCCTrie() : db(GetDataDir() / "ncctrie", 100, false, false), nCurrentHeight(1), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {} CNCCTrie(bool fMemory = false, bool fWipe = false) : db(GetDataDir() / "ncctrie", 100, fMemory, fWipe), nCurrentHeight(1), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {}
uint256 getMerkleHash(); uint256 getMerkleHash();
CLevelDBWrapper db; CLevelDBWrapper db;
bool empty() const; bool empty() const;
bool checkConsistency(); bool checkConsistency();
bool WriteToDisk();
bool ReadFromDisk(bool check = false); bool ReadFromDisk(bool check = false);
json_spirit::Array dumpToJSON() const; json_spirit::Array dumpToJSON() const;
bool getInfoForName(const std::string& name, CNodeValue& val) const; bool getInfoForName(const std::string& name, CNodeValue& val) const;
@ -163,17 +163,20 @@ public:
friend class CNCCTrieCache; friend class CNCCTrieCache;
private: private:
bool update(nodeCacheType& cache, hashMapType& hashes, const uint256& hashBlock, valueQueueType& queueCache, int nNewHeight); bool update(nodeCacheType& cache, hashMapType& hashes, const uint256& hashBlock, valueQueueType& queueCache, int nNewHeight);
bool updateName(const std::string& name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames, CNCCTrieNode** pNodeRet); bool updateName(const std::string& name, CNCCTrieNode* updatedNode);
bool updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet); bool updateHash(const std::string& name, uint256& hash);
bool recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames); bool recursiveNullify(CNCCTrieNode* node, std::string& name);
bool recursiveCheckConsistency(CNCCTrieNode* node); bool recursiveCheckConsistency(CNCCTrieNode* node);
bool BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames, const uint256& hashBlock, std::vector<int> vChangedQueueRows, std::vector<int> vDeletedQueueRows, int nNewHeight);
bool InsertFromDisk(const std::string& name, CNCCTrieNode* node); bool InsertFromDisk(const std::string& name, CNCCTrieNode* node);
bool recursiveDumpToJSON(const std::string& name, const CNCCTrieNode* current, json_spirit::Array& ret) const; bool recursiveDumpToJSON(const std::string& name, const CNCCTrieNode* current, json_spirit::Array& ret) const;
CNCCTrieNode root; CNCCTrieNode root;
uint256 hashBlock; uint256 hashBlock;
valueQueueType valueQueue; valueQueueType valueQueue;
valueQueueType::iterator getQueueRow(int nHeight); valueQueueType::iterator getQueueRow(int nHeight, bool deleteIfNotExists);
nodeCacheType dirtyNodes;
std::vector<int> vDirtyQueueRows;
void markNodeDirty(const std::string& name, CNCCTrieNode* node);
void deleteQueueRow(int nHeight); void deleteQueueRow(int nHeight);
void BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const; void BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const;
void BatchEraseNode(CLevelDBBatch& batch, const std::string& nome) const; void BatchEraseNode(CLevelDBBatch& batch, const std::string& nome) const;

2
src/test/test_bitcoin.cpp
View file

@ -42,7 +42,7 @@ struct TestingSetup {
pblocktree = new CBlockTreeDB(1 << 20, true); pblocktree = new CBlockTreeDB(1 << 20, true);
pcoinsdbview = new CCoinsViewDB(1 << 23, true); pcoinsdbview = new CCoinsViewDB(1 << 23, true);
pcoinsTip = new CCoinsViewCache(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinsdbview);
pnccTrie = new CNCCTrie(); pnccTrie = new CNCCTrie(true);
InitBlockIndex(); InitBlockIndex();
#ifdef ENABLE_WALLET #ifdef ENABLE_WALLET
bool fFirstRun; bool fFirstRun;