flush ncc trie to disk at the same time pcoins and pblocktree are flushed to disk, so that if the program crashes they are at the same block

src/init.cpp

@@ -1046,7 +1046,7 @@ bool AppInit2(boost::thread_group& threadGroup)
                 pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
                 pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
                 pcoinsTip = new CCoinsViewCache(pcoinscatcher);
-                pnccTrie = new CNCCTrie();
+                pnccTrie = new CNCCTrie(false, fReindex);
 
                 if (fReindex)
                     pblocktree->WriteReindexing(true);

src/main.cpp

@@ -2081,6 +2081,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
             }
         }
         // Finally flush the chainstate (which may refer to block index entries).
+        if (!pnccTrie->WriteToDisk())
+            return state.Abort("Failed to write to ncc trie database");
         if (!pcoinsTip->Flush())
             return state.Abort("Failed to write to coin database");
         // Update best block in wallet (so we can detect restored wallets).

src/ncctrie.cpp

@@ -40,7 +40,6 @@ bool CNCCTrieNode::removeValue(uint256& txhash, uint32_t nOut, CNodeValue& val,
 
     CNodeValue currentTop = values.front();
 
-    //std::vector<CNodeValue>::iterator position = std::find(values.begin(), values.end(), val);
     std::vector<CNodeValue>::iterator position;
     for (position = values.begin(); position != values.end(); ++position)
     {
@@ -214,11 +213,13 @@ bool CNCCTrie::recursiveCheckConsistency(CNCCTrieNode* node)
     return calculatedHash == node->hash;
 }
 
-valueQueueType::iterator CNCCTrie::getQueueRow(int nHeight)
+valueQueueType::iterator CNCCTrie::getQueueRow(int nHeight, bool createIfNotExists)
 {
     valueQueueType::iterator itQueueRow = valueQueue.find(nHeight);
     if (itQueueRow == valueQueue.end())
     {
+        if (!createIfNotExists)
+            return itQueueRow;
         std::vector<CValueQueueEntry> queueRow;
         std::pair<valueQueueType::iterator, bool> ret;
         ret = valueQueue.insert(std::pair<int, std::vector<CValueQueueEntry> >(nHeight, queueRow));
@@ -250,56 +251,54 @@ bool CNCCTrie::update(nodeCacheType& cache, hashMapType& hashes, const uint256&
     // This can probably be optimized by checking each substring against
     // the caches each time, but that will come after this is shown to
     // work correctly.
-    // As far as saving to disk goes, the idea is to use the list of
+    // Disk strategy: keep a map of <string: dirty node>, where
-    // hashes to construct a list of (pointers to) nodes that have been
+    // any nodes that are changed get put into the map, and any nodes
-    // altered in the update, and to construct a list of names of nodes
+    // to be deleted will simply be empty (no value, no children). Nodes
-    // that have been deleted, and to use a leveldb batch to write them
+    // whose hashes change will also be inserted into the map.
-    // all to disk. As of right now, txundo stuff will be handled by
+    // As far as the queue goes, just keep a list of dirty queue entries.
-    // appending extra data to the normal txundo, which will call the
+    // When the time comes, send all of that to disk in one batch, and
-    // normal insert/remove names, but obviously the opposite and in
+    // empty the map/list.
-    // reverse order (though the order shouldn't ever matter).
+    
     bool success = true;
-    std::vector<std::string> deletedNames;
-    nodeCacheType changedNodes;
     for (nodeCacheType::iterator itcache = cache.begin(); itcache != cache.end(); ++itcache)
     {
-        CNCCTrieNode* pNode;
+        success = updateName(itcache->first, itcache->second);
-        success = updateName(itcache->first, itcache->second, deletedNames, &pNode);
         if (!success)
             return false;
-        changedNodes[itcache->first] = pNode;
     }
     for (hashMapType::iterator ithash = hashes.begin(); ithash != hashes.end(); ++ithash)
     {
-        CNCCTrieNode* pNode;
+        success = updateHash(ithash->first, ithash->second);
-        success = updateHash(ithash->first, ithash->second, &pNode);
         if (!success)
             return false;
-        changedNodes[ithash->first] = pNode;
     }
-    std::vector<int> vChangedQueueRows;
-    std::vector<int> vDeletedQueueRows;
     for (valueQueueType::iterator itQueueCacheRow = queueCache.begin(); itQueueCacheRow != queueCache.end(); ++itQueueCacheRow)
     {
         if (itQueueCacheRow->second.empty())
         {
-            vDeletedQueueRows.push_back(itQueueCacheRow->first);
             deleteQueueRow(itQueueCacheRow->first);
         }
         else
         {
-            vChangedQueueRows.push_back(itQueueCacheRow->first);
+            valueQueueType::iterator itQueueRow = getQueueRow(itQueueCacheRow->first, true);
-            valueQueueType::iterator itQueueRow = getQueueRow(itQueueCacheRow->first);
             itQueueRow->second.swap(itQueueCacheRow->second);
         }
+        vDirtyQueueRows.push_back(itQueueCacheRow->first);
     }
-    BatchWrite(changedNodes, deletedNames, hashBlockIn, vChangedQueueRows, vDeletedQueueRows, nNewHeight);
     hashBlock = hashBlockIn;
     nCurrentHeight = nNewHeight;
     return true;
 }
 
-bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames, CNCCTrieNode** pNodeRet)
+void CNCCTrie::markNodeDirty(const std::string &name, CNCCTrieNode* node)
+{
+    std::pair<nodeCacheType::iterator, bool> ret;
+    ret = dirtyNodes.insert(std::pair<std::string, CNCCTrieNode*>(name, node));
+    if (ret.second == false)
+        ret.first->second = node;
+}
+
+bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode)
 {
     CNCCTrieNode* current = &root;
     for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
@@ -323,7 +322,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
     }
     assert(current != NULL);
     current->values.swap(updatedNode->values);
-    *pNodeRet = current;
+    markNodeDirty(name, current);
     for (nodeMapType::iterator itchild = current->children.begin(); itchild != current->children.end();)
     {
         nodeMapType::iterator itupdatechild = updatedNode->children.find(itchild->first);
@@ -334,7 +333,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
             std::stringstream ss;
             ss << name << itchild->first;
             std::string newName = ss.str();
-            if (!recursiveNullify(itchild->second, newName, deletedNames))
+            if (!recursiveNullify(itchild->second, newName))
                 return false;
             current->children.erase(itchild++);
         }
@@ -344,7 +343,7 @@ bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, st
     return true;
 }
 
-bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames)
+bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name)
 {
     assert(node != NULL);
     for (nodeMapType::iterator itchild = node->children.begin(); itchild != node->children.end(); ++itchild)
@@ -352,16 +351,16 @@ bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name, std::vect
         std::stringstream ss;
         ss << name << itchild->first;
         std::string newName = ss.str();
-        if (!recursiveNullify(itchild->second, newName, deletedNames))
+        if (!recursiveNullify(itchild->second, newName))
             return false;
     }
     node->children.clear();
+    markNodeDirty(name, NULL);
     delete node;
-    deletedNames.push_back(name);
     return true;
 }
 
-bool CNCCTrie::updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet)
+bool CNCCTrie::updateHash(const std::string& name, uint256& hash)
 {
     CNCCTrieNode* current = &root;
     for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
@@ -373,45 +372,39 @@ bool CNCCTrie::updateHash(const std::string& name, uint256& hash, CNCCTrieNode**
     }
     assert(current != NULL);
     current->hash = hash;
-    *pNodeRet = current;
+    markNodeDirty(name, current);
     return true;
 }
 
 void CNCCTrie::BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const
 {
     LogPrintf("%s: Writing %s to disk with %d values\n", __func__, name, pNode->values.size());
-    batch.Write(std::make_pair('n', name), *pNode);
+    if (pNode)
-}
+        batch.Write(std::make_pair('n', name), *pNode);
-
+    else
-void CNCCTrie::BatchEraseNode(CLevelDBBatch& batch, const std::string& name) const
+        batch.Erase(std::make_pair('n', name));
-{
-    batch.Erase(std::make_pair('n', name));
 }
 
 void CNCCTrie::BatchWriteQueueRow(CLevelDBBatch& batch, int nRowNum)
 {
-    valueQueueType::iterator itQueueRow = getQueueRow(nRowNum);
+    valueQueueType::iterator itQueueRow = getQueueRow(nRowNum, false);
-    batch.Write(std::make_pair('r', nRowNum), itQueueRow->second);
+    if (itQueueRow != valueQueue.end())
+        batch.Write(std::make_pair('r', nRowNum), itQueueRow->second);
+    else
+        batch.Erase(std::make_pair('r', nRowNum));
 }
 
-void CNCCTrie::BatchEraseQueueRow(CLevelDBBatch& batch, int nRowNum)
+bool CNCCTrie::WriteToDisk()
-{
-    batch.Erase(std::make_pair('r', nRowNum));
-}
-
-bool CNCCTrie::BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames, const uint256& hashBlockIn, std::vector<int> vChangedQueueRows, std::vector<int> vDeletedQueueRows, int nNewHeight)
 {
     CLevelDBBatch batch;
-    for (nodeCacheType::iterator itcache = changedNodes.begin(); itcache != changedNodes.end(); ++itcache)
+    for (nodeCacheType::iterator itcache = dirtyNodes.begin(); itcache != dirtyNodes.end(); ++itcache)
         BatchWriteNode(batch, itcache->first, itcache->second);
-    for (std::vector<std::string>::iterator itname = deletedNames.begin(); itname != deletedNames.end(); ++itname)
+    dirtyNodes.clear();
-        BatchEraseNode(batch, *itname);
+    for (std::vector<int>::iterator itRowNum = vDirtyQueueRows.begin(); itRowNum != vDirtyQueueRows.end(); ++itRowNum)
-    for (std::vector<int>::iterator itRowNum = vChangedQueueRows.begin(); itRowNum != vChangedQueueRows.end(); ++itRowNum)
         BatchWriteQueueRow(batch, *itRowNum);
-    for (std::vector<int>::iterator itRowNum = vDeletedQueueRows.begin(); itRowNum != vDeletedQueueRows.end(); ++itRowNum)
+    vDirtyQueueRows.clear();
-        BatchEraseQueueRow(batch, *itRowNum);
+    batch.Write('h', hashBlock);
-    batch.Write('h', hashBlockIn);
+    batch.Write('t', nCurrentHeight);
-    batch.Write('t', nNewHeight);
     return db.WriteBatch(batch);
 }
 
@@ -468,7 +461,7 @@ bool CNCCTrie::ReadFromDisk(bool check)
                 int nHeight;
                 ssKey >> nHeight;
                 CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
-                valueQueueType::iterator itQueueRow = getQueueRow(nHeight);
+                valueQueueType::iterator itQueueRow = getQueueRow(nHeight, true);
                 ssValue >> itQueueRow->second;
             }
             pcursor->Next();

src/ncctrie.h

@@ -26,7 +26,6 @@ public:
     int nHeight;
     int nValidAtHeight;
     CNodeValue() {};
-    //CNodeValue(uint256 txhash, uint32_t nOut) : txhash(txhash), nOut(nOut), nAmount(0), nHeight(0), nValidAtHeight(0) {}
     CNodeValue(uint256 txhash, uint32_t nOut, CAmount nAmount, int nHeight, int nValidAtHeight) : txhash(txhash), nOut(nOut), nAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight) {}
     std::string ToString();
     
@@ -149,11 +148,12 @@ class CNCCTrieCache;
 class CNCCTrie
 {
 public:
-    CNCCTrie() : db(GetDataDir() / "ncctrie", 100, false, false), nCurrentHeight(1), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {}
+    CNCCTrie(bool fMemory = false, bool fWipe = false) : db(GetDataDir() / "ncctrie", 100, fMemory, fWipe), nCurrentHeight(1), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {}
     uint256 getMerkleHash();
     CLevelDBWrapper db;
     bool empty() const;
     bool checkConsistency();
+    bool WriteToDisk();
     bool ReadFromDisk(bool check = false);
     json_spirit::Array dumpToJSON() const;
     bool getInfoForName(const std::string& name, CNodeValue& val) const;
@@ -163,17 +163,20 @@ public:
     friend class CNCCTrieCache;
 private:
     bool update(nodeCacheType& cache, hashMapType& hashes, const uint256& hashBlock, valueQueueType& queueCache, int nNewHeight);
-    bool updateName(const std::string& name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames, CNCCTrieNode** pNodeRet);
+    bool updateName(const std::string& name, CNCCTrieNode* updatedNode);
-    bool updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet);
+    bool updateHash(const std::string& name, uint256& hash);
-    bool recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames);
+    bool recursiveNullify(CNCCTrieNode* node, std::string& name);
     bool recursiveCheckConsistency(CNCCTrieNode* node);
-    bool BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames, const uint256& hashBlock, std::vector<int> vChangedQueueRows, std::vector<int> vDeletedQueueRows, int nNewHeight);
     bool InsertFromDisk(const std::string& name, CNCCTrieNode* node);
     bool recursiveDumpToJSON(const std::string& name, const CNCCTrieNode* current, json_spirit::Array& ret) const;
     CNCCTrieNode root;
     uint256 hashBlock;
     valueQueueType valueQueue;
-    valueQueueType::iterator getQueueRow(int nHeight);
+    valueQueueType::iterator getQueueRow(int nHeight, bool deleteIfNotExists);
+    
+    nodeCacheType dirtyNodes;
+    std::vector<int> vDirtyQueueRows;
+    void markNodeDirty(const std::string& name, CNCCTrieNode* node);
     void deleteQueueRow(int nHeight);
     void BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode) const;
     void BatchEraseNode(CLevelDBBatch& batch, const std::string& nome) const;

src/test/test_bitcoin.cpp

@@ -42,7 +42,7 @@ struct TestingSetup {
         pblocktree = new CBlockTreeDB(1 << 20, true);
         pcoinsdbview = new CCoinsViewDB(1 << 23, true);
         pcoinsTip = new CCoinsViewCache(pcoinsdbview);
-        pnccTrie = new CNCCTrie();
+        pnccTrie = new CNCCTrie(true);
         InitBlockIndex();
 #ifdef ENABLE_WALLET
         bool fFirstRun;