LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

move everything trie related into its own file, enable serialization of trie nodes and values, and create some tests for serialization

Browse source
This commit is contained in:
Jimmy Kiselak 2015-02-02 20:57:53 -05:00
parent e503dbe836
commit 9bb4ecef2a
6 changed files with 798 additions and 643 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Makefile.am
View file

@ -104,6 +104,7 @@ BITCOIN_CORE_H = \
miner.h \
mruset.h \
ncc.h \
ncctrie.h \
netbase.h \
net.h \
noui.h \
@ -173,6 +174,7 @@ libbitcoin_server_a_SOURCES = \
main.cpp \
merkleblock.cpp \
miner.cpp \
ncctrie.cpp \
net.cpp \
noui.cpp \
pow.cpp \

512
src/ncc.cpp
View file

@ -1,5 +1,5 @@
#include "ncc.h"
#include "util.h"
bool DecodeNCCScript(const CScript& scriptIn, int& op, std::vector<std::vector<unsigned char> >& vvchParams)
{
@ -65,513 +65,3 @@ CScript StripNCCScriptPrefix(const CScript& scriptIn)
return CScript(pc, scriptIn.end());
}
std::string CNodeValue::ToString()
{
std::stringstream ss;
ss << nOut;
return txhash.ToString() + ss.str();
}
bool CNCCTrieNode::insertValue(CNodeValue val, bool * pfChanged)
{
bool fChanged = false;
if (values.empty())
{
values.push_back(val);
fChanged = true;
}
else
{
CNodeValue currentTop = values.front();
values.push_back(val);
std::make_heap(values.begin(), values.end());
if (currentTop != values.front())
fChanged = true;
}
if (pfChanged)
*pfChanged = fChanged;
return true;
}
bool CNCCTrieNode::removeValue(CNodeValue val, bool * pfChanged)
{
bool fChanged = false;
CNodeValue currentTop = values.front();
std::vector<CNodeValue>::iterator position = std::find(values.begin(), values.end(), val);
if (position != values.end())
values.erase(position);
else
{
LogPrintf("CNCCTrieNode::removeValue() : asked to remove a value that doesn't exist");
return false;
}
if (!values.empty())
{
std::make_heap(values.begin(), values.end());
if (currentTop != values.front())
fChanged = true;
}
else
fChanged = true;
if (pfChanged)
*pfChanged = fChanged;
return true;
}
bool CNCCTrieNode::getValue(CNodeValue& value)
{
if (values.empty())
return false;
else
{
value = values.front();
return true;
}
}
uint256 CNCCTrie::getMerkleHash()
{
return root.hash;
}
bool CNCCTrie::empty() const
{
return root.empty();
}
bool CNCCTrie::checkConsistency()
{
return recursiveCheckConsistency(&root);
}
bool CNCCTrie::recursiveCheckConsistency(CNCCTrieNode* node)
{
std::string stringToHash;
CNodeValue val;
bool hasValue = node->getValue(val);
if (hasValue)
{
stringToHash += val.ToString();
}
for (nodeMapType::iterator it = node->children.begin(); it != node->children.end(); ++it)
{
std::stringstream ss;
ss << it->first;
if (recursiveCheckConsistency(it->second))
{
stringToHash += ss.str();
stringToHash += it->second->hash.ToString();
}
else
return false;
}
CHash256 hasher;
std::vector<unsigned char> vchHash(hasher.OUTPUT_SIZE);
hasher.Write((const unsigned char*) stringToHash.data(), stringToHash.size());
hasher.Finalize(&(vchHash[0]));
uint256 calculatedHash(vchHash);
return calculatedHash == node->hash;
}
/*bool cachesort (std::pair<std::string, CNCCTrieNode*>& i, std::pair<std::string, CNCCTrieNode*>& j)
{
return i.first.size() < j.first.size();
}*/
bool CNCCTrie::update(nodeCacheType& cache, hashMapType& hashes)
{
// General strategy: the cache is ordered by length, ensuring child
// nodes are always inserted after their parents. Insert each node
// one at a time. When updating a node, swap its values with those
// of the cached node and delete all characters (and their children
// and so forth) which don't exist in the updated node. When adding
// a new node, make sure that its <character, CNCCTrieNode*> pair
// gets into the parent's children.
// Then, update all of the given hashes.
// This can probably be optimized by checking each substring against
// the caches each time, but that will come after this is shown to
// work correctly.
if (cache.empty())
{
return true;
}
//std::sort(cache.begin(), cache.end(), cachesort);
bool success = true;
for (nodeCacheType::iterator itcache = cache.begin(); itcache != cache.end(); ++itcache)
{
success = updateName(itcache->first, itcache->second);
if (!success)
return false;
}
for (hashMapType::iterator ithash = hashes.begin(); ithash != hashes.end(); ++ithash)
{
success = updateHash(ithash->first, ithash->second);
if (!success)
return false;
}
return true;
}
bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode)
{
CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
{
nodeMapType::iterator itchild = current->children.find(*itname);
if (itchild == current->children.end())
{
if (itname + 1 == name.end())
{
CNCCTrieNode* newNode = new CNCCTrieNode();
current->children[*itname] = newNode;
current = newNode;
}
else
return false;
}
else
{
current = itchild->second;
}
}
assert(current != NULL);
bool success = true;
current->values.swap(updatedNode->values);
for (nodeMapType::iterator itchild = current->children.begin(); itchild != current->children.end();)
{
nodeMapType::iterator itupdatechild = updatedNode->children.find(itchild->first);
if (itupdatechild == updatedNode->children.end())
{
// This character has apparently been deleted, so delete
// all descendents from this child.
success = recursiveNullify(itchild->second);
if (!success)
return false;
current->children.erase(itchild++);
}
else
++itchild;
}
return success;
}
bool CNCCTrie::recursiveNullify(CNCCTrieNode* node)
{
assert(node != NULL);
for (nodeMapType::iterator itchild = node->children.begin(); itchild != node->children.end(); ++itchild)
recursiveNullify(itchild->second);
node->children.clear();
delete node;
return true;
}
bool CNCCTrie::updateHash(const std::string& name, uint256& hash)
{
CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
{
nodeMapType::iterator itchild = current->children.find(*itname);
if (itchild == current->children.end())
return false;
current = itchild->second;
}
assert(current != NULL);
current->hash = hash;
return true;
}
bool CNCCTrieCache::recursiveComputeMerkleHash(CNCCTrieNode* tnCurrent, std::string sPos) const
{
std::string stringToHash;
CNodeValue val;
bool hasValue = tnCurrent->getValue(val);
if (hasValue)
{
stringToHash += val.ToString();
}
nodeCacheType::iterator cachedNode;
for (nodeMapType::iterator it = tnCurrent->children.begin(); it != tnCurrent->children.end(); ++it)
{
std::stringstream ss;
ss << it->first;
std::string sNextPos = sPos + ss.str();
if (dirtyHashes.count(sNextPos) != 0)
{
// the child might be in the cache, so look for it there
cachedNode = cache.find(sNextPos);
if (cachedNode != cache.end())
recursiveComputeMerkleHash(cachedNode->second, sNextPos);
else
recursiveComputeMerkleHash(it->second, sNextPos);
}
stringToHash += ss.str();
hashMapType::iterator ithash = cacheHashes.find(sNextPos);
if (ithash != cacheHashes.end())
stringToHash += ithash->second.ToString();
else
stringToHash += it->second->hash.ToString();
}
CHash256 hasher;
std::vector<unsigned char> vchHash(hasher.OUTPUT_SIZE);
hasher.Write((const unsigned char*) stringToHash.data(), stringToHash.size());
hasher.Finalize(&(vchHash[0]));
cacheHashes[sPos] = uint256(vchHash);
std::set<std::string>::iterator itDirty = dirtyHashes.find(sPos);
if (itDirty != dirtyHashes.end())
dirtyHashes.erase(itDirty);
return true;
}
uint256 CNCCTrieCache::getMerkleHash() const
{
if (empty())
{
uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001"));
return one;
}
if (isDirty())
{
nodeCacheType::iterator cachedNode = cache.find("");
if (cachedNode != cache.end())
recursiveComputeMerkleHash(cachedNode->second, "");
else
recursiveComputeMerkleHash(&(base->root), "");
}
hashMapType::iterator ithash = cacheHashes.find("");
if (ithash != cacheHashes.end())
return ithash->second;
else
return base->root.hash;
}
bool CNCCTrieCache::empty() const
{
return base->empty() && cache.empty();
}
bool CNCCTrieCache::insertName(const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nHeight) const
{
CNCCTrieNode* currentNode = &(base->root);
nodeCacheType::iterator cachedNode;
cachedNode = cache.find("");
if (cachedNode != cache.end())
currentNode = cachedNode->second;
if (currentNode == NULL)
{
currentNode = new CNCCTrieNode();
cache[""] = currentNode;
}
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sCurrentSubstring(name.begin(), itCur);
std::string sNextSubstring(name.begin(), itCur + 1);
cachedNode = cache.find(sNextSubstring);
if (cachedNode != cache.end())
{
currentNode = cachedNode->second;
continue;
}
nodeMapType::iterator childNode = currentNode->children.find(*itCur);
if (childNode != currentNode->children.end())
{
currentNode = childNode->second;
continue;
}
// This next substring doesn't exist in the cache and the next
// character doesn't exist in current node's children, so check
// if the current node is in the cache, and if it's not, copy
// it and stick it in the cache, and then create a new node as
// its child and stick that in the cache. We have to have both
// this node and its child in the cache so that the current
// node's child map will contain the next letter, which will be
// used to find the child in the cache. This is necessary in
// order to calculate the merkle hash.
cachedNode = cache.find(sCurrentSubstring);
if (cachedNode != cache.end())
{
assert(cachedNode->second == currentNode);
}
else
{
currentNode = new CNCCTrieNode(*currentNode);
cache[sCurrentSubstring] = currentNode;
}
CNCCTrieNode* newNode = new CNCCTrieNode();
currentNode->children[*itCur] = newNode;
cache[sNextSubstring] = newNode;
currentNode = newNode;
}
cachedNode = cache.find(name);
if (cachedNode != cache.end())
{
assert(cachedNode->second == currentNode);
}
else
{
currentNode = new CNCCTrieNode(*currentNode);
cache[name] = currentNode;
}
bool fChanged = false;
currentNode->insertValue(CNodeValue(txhash, nOut, nAmount, nHeight), &fChanged);
if (fChanged)
{
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sub(name.begin(), itCur);
dirtyHashes.insert(sub);
}
dirtyHashes.insert(name);
}
return true;
}
bool CNCCTrieCache::removeName(const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nHeight) const
{
CNCCTrieNode* currentNode = &(base->root);
nodeCacheType::iterator cachedNode;
cachedNode = cache.find("");
if (cachedNode != cache.end())
currentNode = cachedNode->second;
assert(currentNode != NULL); // If there is no root in either the trie or the cache, how can there be any names to remove?
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sCurrentSubstring(name.begin(), itCur);
std::string sNextSubstring(name.begin(), itCur + 1);
cachedNode = cache.find(sNextSubstring);
if (cachedNode != cache.end())
{
currentNode = cachedNode->second;
continue;
}
nodeMapType::iterator childNode = currentNode->children.find(*itCur);
if (childNode != currentNode->children.end())
{
currentNode = childNode->second;
continue;
}
// The name doesn't exist in either the trie or the cache, so how can we remove it?
return false;
}
cachedNode = cache.find(name);
if (cachedNode != cache.end())
assert(cachedNode->second == currentNode);
else
{
currentNode = new CNCCTrieNode(*currentNode);
cache[name] = currentNode;
}
bool fChanged = false;
assert(currentNode != NULL);
bool success = currentNode->removeValue(CNodeValue(txhash, nOut, nAmount, nHeight), &fChanged);
assert(success);
if (fChanged)
{
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sub(name.begin(), itCur);
dirtyHashes.insert(sub);
}
dirtyHashes.insert(name);
}
CNCCTrieNode* rootNode = &(base->root);
cachedNode = cache.find("");
if (cachedNode != cache.end())
rootNode = cachedNode->second;
return recursivePruneName(rootNode, 0, name);
}
bool CNCCTrieCache::recursivePruneName(CNCCTrieNode* tnCurrent, unsigned int nPos, std::string sName, bool* pfNullified) const
{
bool fNullified = false;
std::string sCurrentSubstring = sName.substr(0, nPos);
if (nPos < sName.size())
{
std::string sNextSubstring = sName.substr(0, nPos + 1);
unsigned char cNext = sName.at(nPos);
CNCCTrieNode* tnNext = NULL;
nodeCacheType::iterator cachedNode = cache.find(sNextSubstring);
if (cachedNode != cache.end())
tnNext = cachedNode->second;
else
{
nodeMapType::iterator childNode = tnCurrent->children.find(cNext);
if (childNode != tnCurrent->children.end())
tnNext = childNode->second;
}
if (tnNext == NULL)
return false;
bool fChildNullified = false;
if (!recursivePruneName(tnNext, nPos + 1, sName, &fChildNullified))
return false;
if (fChildNullified)
{
// If the child nullified itself, the child should already be
// out of the cache, and the character must now be removed
// from the current node's map of child nodes to ensure that
// it isn't found when calculating the merkle hash. But
// tnCurrent isn't necessarily in the cache. If it's not, it
// has to be added to the cache, so nothing is changed in the
// trie. If the current node is added to the cache, however,
// that does not imply that the parent node must be altered to
// reflect that its child is now in the cache, since it
// already has a character in its child map which will be used
// when calculating the merkle root.
// First, find out if this node is in the cache.
cachedNode = cache.find(sCurrentSubstring);
if (cachedNode == cache.end())
{
// it isn't, so make a copy, stick it in the cache,
// and make it the new current node
tnCurrent = new CNCCTrieNode(*tnCurrent);
cache[sCurrentSubstring] = tnCurrent;
}
// erase the character from the current node, which is
// now guaranteed to be in the cache
nodeMapType::iterator childNode = tnCurrent->children.find(cNext);
if (childNode != tnCurrent->children.end())
tnCurrent->children.erase(childNode);
else
return false;
}
}
if (sCurrentSubstring.size() != 0 && tnCurrent->empty())
{
// If the current node is in the cache, remove it from there
nodeCacheType::iterator cachedNode = cache.find(sCurrentSubstring);
if (cachedNode != cache.end())
{
assert(tnCurrent == cachedNode->second);
delete tnCurrent;
cache.erase(cachedNode);
}
fNullified = true;
}
if (pfNullified)
*pfNullified = fNullified;
return true;
}
bool CNCCTrieCache::Flush()
{
if (isDirty())
getMerkleHash();
return base->update(cache, cacheHashes);
}

126
src/ncc.h
View file

@ -2,135 +2,11 @@
#define BITCOIN_NCC_H
#include "script/script.h"
#include "amount.h"
#include "primitives/transaction.h"
#include "coins.h"
#include "hash.h"
#include "uint256.h"
#include "util.h"
#include <iostream>
#include <string>
#include <algorithm>
#include <vector>
bool DecodeNCCScript(const CScript& scriptIn, int& op, std::vector<std::vector<unsigned char> >& vvchParams);
bool DecodeNCCScript(const CScript& scriptIn, int& op, std::vector<std::vector<unsigned char> >& vvchParams, CScript::const_iterator& pc);
CScript StripNCCScriptPrefix(const CScript& scriptIn);
class CNodeValue
{
public:
uint256 txhash;
uint32_t nOut;
CAmount nAmount;
int nHeight;
CNodeValue() {};
CNodeValue(uint256 txhash, uint32_t nOut, CAmount nAmount, int nHeight) : txhash(txhash), nOut(nOut), nAmount(nAmount), nHeight(nHeight) {}
std::string ToString();
bool operator<(const CNodeValue& other)
{
if (nAmount < other.nAmount)
return true;
else if (nAmount == other.nAmount)
{
if (nHeight > other.nHeight)
return true;
else if (nHeight == other.nHeight)
{
if (txhash.GetHex() > other.txhash.GetHex())
return true;
else if (txhash == other.txhash)
if (nOut > other.nOut)
return true;
}
}
return false;
}
bool operator==(const CNodeValue& other)
{
return txhash == other.txhash && nOut == other.nOut && nAmount == other.nAmount && nHeight == other.nHeight;
}
bool operator!=(const CNodeValue& other)
{
return !(*this == other);
}
};
class CNCCTrieNode;
class CNCCTrie;
typedef std::map<unsigned char, CNCCTrieNode*> nodeMapType;
class CNCCTrieNode
{
public:
CNCCTrieNode() {}
CNCCTrieNode(uint256 hash) : hash(hash) {}
uint256 hash;
uint256 bestBlock;
nodeMapType children;
bool insertValue(CNodeValue val, bool * fChanged);
bool removeValue(CNodeValue val, bool * fChanged);
bool getValue(CNodeValue& val);
bool empty() const {return children.empty() && values.empty();}
friend class CNCCTrie;
private:
std::vector<CNodeValue> values;
};
struct nodenamecompare
{
bool operator() (const std::string& i, const std::string& j) const
{
if (i.size() == j.size())
return i < j;
return i.size() < j.size();
}
};
typedef std::map<std::string, CNCCTrieNode*, nodenamecompare> nodeCacheType;
typedef std::map<std::string, uint256> hashMapType;
class CNCCTrieCache;
class CNCCTrie
{
public:
CNCCTrie(CCoinsViewCache* coins) : coins(coins), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {}
uint256 getMerkleHash();
bool empty() const;
bool checkConsistency();
friend class CNCCTrieCache;
private:
CCoinsViewCache* coins;
bool update(nodeCacheType& cache, hashMapType& hashes);
bool updateName(const std::string& name, CNCCTrieNode* updatedNode);
bool updateHash(const std::string& name, uint256& hash);
bool recursiveNullify(CNCCTrieNode* node);
bool recursiveCheckConsistency(CNCCTrieNode* node);
CNCCTrieNode root;
};
class CNCCTrieCache
{
public:
CNCCTrieCache(CNCCTrie* base): base(base) {}
uint256 getMerkleHash() const;
bool empty() const;
bool Flush();
bool isDirty() const { return !dirtyHashes.empty(); }
bool insertName (const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nDepth) const;
bool removeName (const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nDepth) const;
private:
CNCCTrie* base;
mutable nodeCacheType cache;
mutable std::set<std::string> dirtyHashes;
mutable hashMapType cacheHashes;
uint256 computeHash() const;
bool recursiveComputeMerkleHash(CNCCTrieNode* tnCurrent, std::string sPos) const;
bool recursivePruneName(CNCCTrieNode* tnCurrent, unsigned int nPos, std::string sName, bool* pfNullified = NULL) const;
};
#endif // BITCOIN_NCC_H

571
src/ncctrie.cpp Normal file
View file

@ -0,0 +1,571 @@
#include "ncctrie.h"
std::string CNodeValue::ToString()
{
std::stringstream ss;
ss << nOut;
return txhash.ToString() + ss.str();
}
bool CNCCTrieNode::insertValue(CNodeValue val, bool * pfChanged)
{
bool fChanged = false;
if (values.empty())
{
values.push_back(val);
fChanged = true;
}
else
{
CNodeValue currentTop = values.front();
values.push_back(val);
std::make_heap(values.begin(), values.end());
if (currentTop != values.front())
fChanged = true;
}
if (pfChanged)
*pfChanged = fChanged;
return true;
}
bool CNCCTrieNode::removeValue(CNodeValue val, bool * pfChanged)
{
bool fChanged = false;
CNodeValue currentTop = values.front();
std::vector<CNodeValue>::iterator position = std::find(values.begin(), values.end(), val);
if (position != values.end())
values.erase(position);
else
{
LogPrintf("CNCCTrieNode::removeValue() : asked to remove a value that doesn't exist");
return false;
}
if (!values.empty())
{
std::make_heap(values.begin(), values.end());
if (currentTop != values.front())
fChanged = true;
}
else
fChanged = true;
if (pfChanged)
*pfChanged = fChanged;
return true;
}
bool CNCCTrieNode::getValue(CNodeValue& value)
{
if (values.empty())
return false;
else
{
value = values.front();
return true;
}
}
uint256 CNCCTrie::getMerkleHash()
{
return root.hash;
}
bool CNCCTrie::empty() const
{
return root.empty();
}
bool CNCCTrie::checkConsistency()
{
return recursiveCheckConsistency(&root);
}
bool CNCCTrie::recursiveCheckConsistency(CNCCTrieNode* node)
{
std::string stringToHash;
CNodeValue val;
bool hasValue = node->getValue(val);
if (hasValue)
{
stringToHash += val.ToString();
}
for (nodeMapType::iterator it = node->children.begin(); it != node->children.end(); ++it)
{
std::stringstream ss;
ss << it->first;
if (recursiveCheckConsistency(it->second))
{
stringToHash += ss.str();
stringToHash += it->second->hash.ToString();
}
else
return false;
}
CHash256 hasher;
std::vector<unsigned char> vchHash(hasher.OUTPUT_SIZE);
hasher.Write((const unsigned char*) stringToHash.data(), stringToHash.size());
hasher.Finalize(&(vchHash[0]));
uint256 calculatedHash(vchHash);
return calculatedHash == node->hash;
}
/*bool cachesort (std::pair<std::string, CNCCTrieNode*>& i, std::pair<std::string, CNCCTrieNode*>& j)
{
return i.first.size() < j.first.size();
}*/
bool CNCCTrie::update(nodeCacheType& cache, hashMapType& hashes)
{
// General strategy: the cache is ordered by length, ensuring child
// nodes are always inserted after their parents. Insert each node
// one at a time. When updating a node, swap its values with those
// of the cached node and delete all characters (and their children
// and so forth) which don't exist in the updated node. When adding
// a new node, make sure that its <character, CNCCTrieNode*> pair
// gets into the parent's children.
// Then, update all of the given hashes.
// This can probably be optimized by checking each substring against
// the caches each time, but that will come after this is shown to
// work correctly.
// As far as saving to disk goes, the idea is to use the list of
// hashes to construct a list of (pointers to) nodes that have been
// altered in the update, and to construct a list of names of nodes
// that have been deleted, and to use a leveldb batch to write them
// all to disk. As of right now, txundo stuff will be handled by
// appending extra data to the normal txundo, which will call the
// normal insert/remove names, but obviously the opposite and in
// reverse order (though the order shouldn't ever matter).
if (cache.empty())
{
return true;
}
//std::sort(cache.begin(), cache.end(), cachesort);
bool success = true;
std::vector<std::string> deletedNames;
for (nodeCacheType::iterator itcache = cache.begin(); itcache != cache.end(); ++itcache)
{
success = updateName(itcache->first, itcache->second, deletedNames);
if (!success)
return false;
}
nodeCacheType changedNodes;
for (hashMapType::iterator ithash = hashes.begin(); ithash != hashes.end(); ++ithash)
{
CNCCTrieNode* pNode;
success = updateHash(ithash->first, ithash->second, &pNode);
if (!success)
return false;
changedNodes[ithash->first] = pNode;
}
BatchWrite(changedNodes, deletedNames);
return true;
}
bool CNCCTrie::updateName(const std::string &name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames)
{
CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
{
nodeMapType::iterator itchild = current->children.find(*itname);
if (itchild == current->children.end())
{
if (itname + 1 == name.end())
{
CNCCTrieNode* newNode = new CNCCTrieNode();
current->children[*itname] = newNode;
current = newNode;
}
else
return false;
}
else
{
current = itchild->second;
}
}
assert(current != NULL);
current->values.swap(updatedNode->values);
for (nodeMapType::iterator itchild = current->children.begin(); itchild != current->children.end();)
{
nodeMapType::iterator itupdatechild = updatedNode->children.find(itchild->first);
if (itupdatechild == updatedNode->children.end())
{
// This character has apparently been deleted, so delete
// all descendents from this child.
std::stringstream ss;
ss << name << itchild->first;
std::string newName = ss.str();
if (!recursiveNullify(itchild->second, newName, deletedNames))
return false;
current->children.erase(itchild++);
}
else
++itchild;
}
return true;
}
bool CNCCTrie::recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames)
{
assert(node != NULL);
for (nodeMapType::iterator itchild = node->children.begin(); itchild != node->children.end(); ++itchild)
{
std::stringstream ss;
ss << name << itchild->first;
std::string newName = ss.str();
if (!recursiveNullify(itchild->second, newName, deletedNames))
return false;
}
node->children.clear();
delete node;
deletedNames.push_back(name);
return true;
}
bool CNCCTrie::updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet)
{
CNCCTrieNode* current = &root;
for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname)
{
nodeMapType::iterator itchild = current->children.find(*itname);
if (itchild == current->children.end())
return false;
current = itchild->second;
}
assert(current != NULL);
current->hash = hash;
*pNodeRet = current;
return true;
}
void BatchWriteNode(CLevelDBBatch& batch, const std::string& name, const CNCCTrieNode* pNode)
{
batch.Write(std::make_pair('n', name), *pNode);
}
void BatchEraseNode(CLevelDBBatch& batch, const std::string& name)
{
batch.Erase(std::make_pair('n', name));
}
bool CNCCTrie::BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames)
{
CLevelDBBatch batch;
for (nodeCacheType::iterator itcache = changedNodes.begin(); itcache != changedNodes.end(); ++itcache)
BatchWriteNode(batch, itcache->first, itcache->second);
for (std::vector<std::string>::iterator itname = deletedNames.begin(); itname != deletedNames.end(); ++itname)
BatchEraseNode(batch, *itname);
return db.WriteBatch(batch);
}
bool CNCCTrieCache::recursiveComputeMerkleHash(CNCCTrieNode* tnCurrent, std::string sPos) const
{
std::string stringToHash;
CNodeValue val;
bool hasValue = tnCurrent->getValue(val);
if (hasValue)
{
stringToHash += val.ToString();
}
nodeCacheType::iterator cachedNode;
for (nodeMapType::iterator it = tnCurrent->children.begin(); it != tnCurrent->children.end(); ++it)
{
std::stringstream ss;
ss << it->first;
std::string sNextPos = sPos + ss.str();
if (dirtyHashes.count(sNextPos) != 0)
{
// the child might be in the cache, so look for it there
cachedNode = cache.find(sNextPos);
if (cachedNode != cache.end())
recursiveComputeMerkleHash(cachedNode->second, sNextPos);
else
recursiveComputeMerkleHash(it->second, sNextPos);
}
stringToHash += ss.str();
hashMapType::iterator ithash = cacheHashes.find(sNextPos);
if (ithash != cacheHashes.end())
stringToHash += ithash->second.ToString();
else
stringToHash += it->second->hash.ToString();
}
CHash256 hasher;
std::vector<unsigned char> vchHash(hasher.OUTPUT_SIZE);
hasher.Write((const unsigned char*) stringToHash.data(), stringToHash.size());
hasher.Finalize(&(vchHash[0]));
cacheHashes[sPos] = uint256(vchHash);
std::set<std::string>::iterator itDirty = dirtyHashes.find(sPos);
if (itDirty != dirtyHashes.end())
dirtyHashes.erase(itDirty);
return true;
}
uint256 CNCCTrieCache::getMerkleHash() const
{
if (empty())
{
uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001"));
return one;
}
if (dirty())
{
nodeCacheType::iterator cachedNode = cache.find("");
if (cachedNode != cache.end())
recursiveComputeMerkleHash(cachedNode->second, "");
else
recursiveComputeMerkleHash(&(base->root), "");
}
hashMapType::iterator ithash = cacheHashes.find("");
if (ithash != cacheHashes.end())
return ithash->second;
else
return base->root.hash;
}
bool CNCCTrieCache::empty() const
{
return base->empty() && cache.empty();
}
bool CNCCTrieCache::insertName(const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nHeight) const
{
CNCCTrieNode* currentNode = &(base->root);
nodeCacheType::iterator cachedNode;
cachedNode = cache.find("");
if (cachedNode != cache.end())
currentNode = cachedNode->second;
if (currentNode == NULL)
{
currentNode = new CNCCTrieNode();
cache[""] = currentNode;
}
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sCurrentSubstring(name.begin(), itCur);
std::string sNextSubstring(name.begin(), itCur + 1);
cachedNode = cache.find(sNextSubstring);
if (cachedNode != cache.end())
{
currentNode = cachedNode->second;
continue;
}
nodeMapType::iterator childNode = currentNode->children.find(*itCur);
if (childNode != currentNode->children.end())
{
currentNode = childNode->second;
continue;
}
// This next substring doesn't exist in the cache and the next
// character doesn't exist in current node's children, so check
// if the current node is in the cache, and if it's not, copy
// it and stick it in the cache, and then create a new node as
// its child and stick that in the cache. We have to have both
// this node and its child in the cache so that the current
// node's child map will contain the next letter, which will be
// used to find the child in the cache. This is necessary in
// order to calculate the merkle hash.
cachedNode = cache.find(sCurrentSubstring);
if (cachedNode != cache.end())
{
assert(cachedNode->second == currentNode);
}
else
{
currentNode = new CNCCTrieNode(*currentNode);
cache[sCurrentSubstring] = currentNode;
}
CNCCTrieNode* newNode = new CNCCTrieNode();
currentNode->children[*itCur] = newNode;
cache[sNextSubstring] = newNode;
currentNode = newNode;
}
cachedNode = cache.find(name);
if (cachedNode != cache.end())
{
assert(cachedNode->second == currentNode);
}
else
{
currentNode = new CNCCTrieNode(*currentNode);
cache[name] = currentNode;
}
bool fChanged = false;
currentNode->insertValue(CNodeValue(txhash, nOut, nAmount, nHeight), &fChanged);
if (fChanged)
{
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sub(name.begin(), itCur);
dirtyHashes.insert(sub);
}
dirtyHashes.insert(name);
}
return true;
}
bool CNCCTrieCache::removeName(const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nHeight) const
{
CNCCTrieNode* currentNode = &(base->root);
nodeCacheType::iterator cachedNode;
cachedNode = cache.find("");
if (cachedNode != cache.end())
currentNode = cachedNode->second;
assert(currentNode != NULL); // If there is no root in either the trie or the cache, how can there be any names to remove?
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sCurrentSubstring(name.begin(), itCur);
std::string sNextSubstring(name.begin(), itCur + 1);
cachedNode = cache.find(sNextSubstring);
if (cachedNode != cache.end())
{
currentNode = cachedNode->second;
continue;
}
nodeMapType::iterator childNode = currentNode->children.find(*itCur);
if (childNode != currentNode->children.end())
{
currentNode = childNode->second;
continue;
}
// The name doesn't exist in either the trie or the cache, so how can we remove it?
return false;
}
cachedNode = cache.find(name);
if (cachedNode != cache.end())
assert(cachedNode->second == currentNode);
else
{
currentNode = new CNCCTrieNode(*currentNode);
cache[name] = currentNode;
}
bool fChanged = false;
assert(currentNode != NULL);
bool success = currentNode->removeValue(CNodeValue(txhash, nOut, nAmount, nHeight), &fChanged);
assert(success);
if (fChanged)
{
for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur)
{
std::string sub(name.begin(), itCur);
dirtyHashes.insert(sub);
}
dirtyHashes.insert(name);
}
CNCCTrieNode* rootNode = &(base->root);
cachedNode = cache.find("");
if (cachedNode != cache.end())
rootNode = cachedNode->second;
return recursivePruneName(rootNode, 0, name);
}
bool CNCCTrieCache::recursivePruneName(CNCCTrieNode* tnCurrent, unsigned int nPos, std::string sName, bool* pfNullified) const
{
bool fNullified = false;
std::string sCurrentSubstring = sName.substr(0, nPos);
if (nPos < sName.size())
{
std::string sNextSubstring = sName.substr(0, nPos + 1);
unsigned char cNext = sName.at(nPos);
CNCCTrieNode* tnNext = NULL;
nodeCacheType::iterator cachedNode = cache.find(sNextSubstring);
if (cachedNode != cache.end())
tnNext = cachedNode->second;
else
{
nodeMapType::iterator childNode = tnCurrent->children.find(cNext);
if (childNode != tnCurrent->children.end())
tnNext = childNode->second;
}
if (tnNext == NULL)
return false;
bool fChildNullified = false;
if (!recursivePruneName(tnNext, nPos + 1, sName, &fChildNullified))
return false;
if (fChildNullified)
{
// If the child nullified itself, the child should already be
// out of the cache, and the character must now be removed
// from the current node's map of child nodes to ensure that
// it isn't found when calculating the merkle hash. But
// tnCurrent isn't necessarily in the cache. If it's not, it
// has to be added to the cache, so nothing is changed in the
// trie. If the current node is added to the cache, however,
// that does not imply that the parent node must be altered to
// reflect that its child is now in the cache, since it
// already has a character in its child map which will be used
// when calculating the merkle root.
// First, find out if this node is in the cache.
cachedNode = cache.find(sCurrentSubstring);
if (cachedNode == cache.end())
{
// it isn't, so make a copy, stick it in the cache,
// and make it the new current node
tnCurrent = new CNCCTrieNode(*tnCurrent);
cache[sCurrentSubstring] = tnCurrent;
}
// erase the character from the current node, which is
// now guaranteed to be in the cache
nodeMapType::iterator childNode = tnCurrent->children.find(cNext);
if (childNode != tnCurrent->children.end())
tnCurrent->children.erase(childNode);
else
return false;
}
}
if (sCurrentSubstring.size() != 0 && tnCurrent->empty())
{
// If the current node is in the cache, remove it from there
nodeCacheType::iterator cachedNode = cache.find(sCurrentSubstring);
if (cachedNode != cache.end())
{
assert(tnCurrent == cachedNode->second);
delete tnCurrent;
cache.erase(cachedNode);
}
fNullified = true;
}
if (pfNullified)
*pfNullified = fNullified;
return true;
}
bool CNCCTrieCache::clear() const
{
for (nodeCacheType::iterator itcache = cache.begin(); itcache != cache.end(); ++itcache)
{
delete itcache->second;
}
cache.clear();
dirtyHashes.clear();
cacheHashes.clear();
return true;
}
bool CNCCTrieCache::flush()
{
if (dirty())
getMerkleHash();
bool success = base->update(cache, cacheHashes);
if (success)
{
success = clear();
}
return success;
}

164
src/ncctrie.h Normal file
View file

@ -0,0 +1,164 @@
#ifndef BITCOIN_NCCTRIE_H
#define BITCOIN_NCC_H
#include "amount.h"
#include "serialize.h"
#include "coins.h"
#include "hash.h"
#include "uint256.h"
#include "util.h"
#include "leveldbwrapper.h"
#include <iostream>
#include <string>
#include <algorithm>
#include <vector>
class CNodeValue
{
public:
uint256 txhash;
uint32_t nOut;
CAmount nAmount;
int nHeight;
CNodeValue() {};
CNodeValue(uint256 txhash, uint32_t nOut, CAmount nAmount, int nHeight) : txhash(txhash), nOut(nOut), nAmount(nAmount), nHeight(nHeight) {}
std::string ToString();
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
READWRITE(txhash);
READWRITE(nOut);
READWRITE(nAmount);
READWRITE(nHeight);
}
bool operator<(const CNodeValue& other) const
{
if (nAmount < other.nAmount)
return true;
else if (nAmount == other.nAmount)
{
if (nHeight > other.nHeight)
return true;
else if (nHeight == other.nHeight)
{
if (txhash.GetHex() > other.txhash.GetHex())
return true;
else if (txhash == other.txhash)
if (nOut > other.nOut)
return true;
}
}
return false;
}
bool operator==(const CNodeValue& other) const
{
return txhash == other.txhash && nOut == other.nOut && nAmount == other.nAmount && nHeight == other.nHeight;
}
bool operator!=(const CNodeValue& other) const
{
return !(*this == other);
}
};
class CNCCTrieNode;
class CNCCTrie;
typedef std::map<unsigned char, CNCCTrieNode*> nodeMapType;
class CNCCTrieNode
{
public:
CNCCTrieNode() {}
CNCCTrieNode(uint256 hash) : hash(hash) {}
uint256 hash;
uint256 bestBlock;
nodeMapType children;
std::vector<CNodeValue> values;
bool insertValue(CNodeValue val, bool * fChanged = NULL);
bool removeValue(CNodeValue val, bool * fChanged = NULL);
bool getValue(CNodeValue& val);
bool empty() const {return children.empty() && values.empty();}
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
READWRITE(hash);
READWRITE(values);
}
bool operator==(const CNCCTrieNode& other) const
{
return hash == other.hash && values == other.values;
}
bool operator!=(const CNCCTrieNode& other) const
{
return !(*this == other);
}
};
struct nodenamecompare
{
bool operator() (const std::string& i, const std::string& j) const
{
if (i.size() == j.size())
return i < j;
return i.size() < j.size();
}
};
typedef std::map<std::string, CNCCTrieNode*, nodenamecompare> nodeCacheType;
typedef std::map<std::string, uint256> hashMapType;
class CNCCTrieCache;
class CNCCTrie
{
public:
CNCCTrie(CCoinsViewCache* coins) : db(GetDataDir() / "ncctrie", 100, false, false), coins(coins), root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {}
uint256 getMerkleHash();
CLevelDBWrapper db;
bool empty() const;
bool checkConsistency();
friend class CNCCTrieCache;
private:
CCoinsViewCache* coins;
bool update(nodeCacheType& cache, hashMapType& hashes);
bool updateName(const std::string& name, CNCCTrieNode* updatedNode, std::vector<std::string>& deletedNames);
bool updateHash(const std::string& name, uint256& hash, CNCCTrieNode** pNodeRet);
bool recursiveNullify(CNCCTrieNode* node, std::string& name, std::vector<std::string>& deletedNames);
bool recursiveCheckConsistency(CNCCTrieNode* node);
bool BatchWrite(nodeCacheType& changedNodes, std::vector<std::string>& deletedNames);
CNCCTrieNode root;
};
class CNCCTrieCache
{
public:
CNCCTrieCache(CNCCTrie* base): base(base) {}
uint256 getMerkleHash() const;
bool empty() const;
bool flush();
bool dirty() const { return !dirtyHashes.empty(); }
bool insertName (const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nDepth) const;
bool removeName (const std::string name, uint256 txhash, int nOut, CAmount nAmount, int nDepth) const;
~CNCCTrieCache() { clear(); }
private:
CNCCTrie* base;
mutable nodeCacheType cache;
mutable std::set<std::string> dirtyHashes;
mutable hashMapType cacheHashes;
uint256 computeHash() const;
bool recursiveComputeMerkleHash(CNCCTrieNode* tnCurrent, std::string sPos) const;
bool recursivePruneName(CNCCTrieNode* tnCurrent, unsigned int nPos, std::string sName, bool* pfNullified = NULL) const;
bool clear() const;
};
#endif // BITCOIN_NCCTRIE_H

66
src/test/ncctrie_tests.cpp
View file

@ -3,8 +3,9 @@
// file COPYING or http://opensource.org/licenses/mit-license.php
#include "primitives/transaction.h"
#include "ncc.h"
#include "ncctrie.h"
#include "coins.h"
#include "streams.h"
#include <boost/test/unit_test.hpp>
#include <iostream>
@ -72,7 +73,7 @@ BOOST_AUTO_TEST_CASE(ncctrie_create_insert_remov)
ntState.insertName(std::string("testtesttesttest"), tx5.GetHash(), 0, 50, 100);
ntState.removeName(std::string("testtesttesttest"), tx5.GetHash(), 0, 50, 100);
BOOST_CHECK(ntState.getMerkleHash() == hash2);
ntState.Flush();
ntState.flush();
BOOST_CHECK(!trie.empty());
BOOST_CHECK(trie.getMerkleHash() == hash2);
@ -83,7 +84,7 @@ BOOST_AUTO_TEST_CASE(ncctrie_create_insert_remov)
BOOST_CHECK(ntState2.getMerkleHash() == hash3);
ntState2.Flush();
ntState2.flush();
BOOST_CHECK(!trie.empty());
BOOST_CHECK(trie.getMerkleHash() == hash3);
@ -92,7 +93,7 @@ BOOST_AUTO_TEST_CASE(ncctrie_create_insert_remov)
CNCCTrieCache ntState3(&trie);
ntState3.insertName(std::string("test"), tx1.GetHash(), 0, 50, 100);
BOOST_CHECK(ntState3.getMerkleHash() == hash4);
ntState3.Flush();
ntState3.flush();
BOOST_CHECK(!trie.empty());
BOOST_CHECK(trie.getMerkleHash() == hash4);
BOOST_CHECK(trie.checkConsistency());
@ -100,7 +101,7 @@ BOOST_AUTO_TEST_CASE(ncctrie_create_insert_remov)
CNCCTrieCache ntState4(&trie);
ntState4.removeName(std::string("abab"), tx6.GetHash(), 0, 50, 100);
BOOST_CHECK(ntState4.getMerkleHash() == hash2);
ntState4.Flush();
ntState4.flush();
BOOST_CHECK(!trie.empty());
BOOST_CHECK(trie.getMerkleHash() == hash2);
BOOST_CHECK(trie.checkConsistency());
@ -109,7 +110,7 @@ BOOST_AUTO_TEST_CASE(ncctrie_create_insert_remov)
ntState5.removeName(std::string("test"), tx3.GetHash(), 0, 50, 101);
BOOST_CHECK(ntState5.getMerkleHash() == hash2);
ntState5.Flush();
ntState5.flush();
BOOST_CHECK(!trie.empty());
BOOST_CHECK(trie.getMerkleHash() == hash2);
BOOST_CHECK(trie.checkConsistency());
@ -118,10 +119,61 @@ BOOST_AUTO_TEST_CASE(ncctrie_create_insert_remov)
ntState6.insertName(std::string("test"), tx3.GetHash(), 0, 50, 101);
BOOST_CHECK(ntState6.getMerkleHash() == hash2);
ntState6.Flush();
ntState6.flush();
BOOST_CHECK(!trie.empty());
BOOST_CHECK(trie.getMerkleHash() == hash2);
BOOST_CHECK(trie.checkConsistency());
}
BOOST_AUTO_TEST_CASE(ncctrienode_serialize_unserialize)
{
CDataStream ss(SER_DISK, 0);
CNCCTrieNode n1;
CNCCTrieNode n2;
ss << n1;
ss >> n2;
BOOST_CHECK(n1 == n2);
n1.hash.SetHex("0000000000000000000000000000000000000000000000000000000000000001");
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK(n1 == n2);
n1.hash.SetHex("a79e8a5b28f7fa5e8836a4b48da9988bdf56ce749f81f413cb754f963a516200");
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK(n1 == n2);
CNodeValue v1(uint256S("0000000000000000000000000000000000000000000000000000000000000001"), 0, 50, 0);
CNodeValue v2(uint256S("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), 1, 100, 1);
n1.insertValue(v1);
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK(n1 == n2);
n1.insertValue(v2);
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK(n1 == n2);
n1.removeValue(v1);
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK(n1 == n2);
n1.removeValue(v2);
BOOST_CHECK(n1 != n2);
ss << n1;
ss >> n2;
BOOST_CHECK(n1 == n2);
}
BOOST_AUTO_TEST_SUITE_END()