LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

still fixing tests

Browse source
This commit is contained in:
Brannon King 2019-10-25 23:34:52 -06:00
parent 86ae72abb2
commit e96c393c34
7 changed files with 183 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
src/claimtrie.cpp
View file

@ -46,15 +46,15 @@ static const sqlite::sqlite_config sharedConfig{
nullptr, sqlite::Encoding::UTF8 nullptr, sqlite::Encoding::UTF8
}; };
CClaimTrie::CClaimTrie(bool fWipe, int height, int proportionalDelayFactor, std::size_t cacheMB) CClaimTrie::CClaimTrie(bool fWipe, int height, int proportionalDelayFactor)
: dbPath((GetDataDir() / "claims.sqlite").string()), db(dbPath, sharedConfig), : dbPath((GetDataDir() / "claims.sqlite").string()), db(dbPath, sharedConfig),
nNextHeight(height), nProportionalDelayFactor(proportionalDelayFactor) nNextHeight(height), nProportionalDelayFactor(proportionalDelayFactor)
{ {
db.define("merkle_root", [](std::vector<uint256>& hashes, const std::vector<unsigned char>& blob) { hashes.emplace_back(uint256(blob)); }, db.define("MERKLE_ROOT", [](std::vector<uint256>& hashes, const std::vector<unsigned char>& blob) { hashes.emplace_back(uint256(blob)); },
[](const std::vector<uint256>& hashes) { return ComputeMerkleRoot(hashes); }); [](const std::vector<uint256>& hashes) { return ComputeMerkleRoot(hashes); });
db.define("merkle_pair", [](const std::vector<unsigned char>& blob1, const std::vector<unsigned char>& blob2) { return Hash(blob1.begin(), blob1.end(), blob2.begin(), blob2.end()); }); db.define("MERKLE_PAIR", [](const std::vector<unsigned char>& blob1, const std::vector<unsigned char>& blob2) { return Hash(blob1.begin(), blob1.end(), blob2.begin(), blob2.end()); });
db.define("merkle", [](const std::vector<unsigned char>& blob1) { return Hash(blob1.begin(), blob1.end()); }); db.define("MERKLE", [](const std::vector<unsigned char>& blob1) { return Hash(blob1.begin(), blob1.end()); });
db << "CREATE TABLE IF NOT EXISTS nodes (name TEXT NOT NULL PRIMARY KEY, parent TEXT, hash BLOB)"; db << "CREATE TABLE IF NOT EXISTS nodes (name TEXT NOT NULL PRIMARY KEY, parent TEXT, hash BLOB)";
db << "CREATE INDEX IF NOT EXISTS nodes_hash ON nodes (hash)"; db << "CREATE INDEX IF NOT EXISTS nodes_hash ON nodes (hash)";
@ -78,9 +78,10 @@ CClaimTrie::CClaimTrie(bool fWipe, int height, int proportionalDelayFactor, std:
db << "CREATE INDEX IF NOT EXISTS supports_expirationHeight ON supports (expirationHeight)"; db << "CREATE INDEX IF NOT EXISTS supports_expirationHeight ON supports (expirationHeight)";
db << "CREATE INDEX IF NOT EXISTS supports_nodeName ON supports (nodeName)"; db << "CREATE INDEX IF NOT EXISTS supports_nodeName ON supports (nodeName)";
db << "PRAGMA cache_size=-" + std::to_string(cacheMB * 1024); // in -KB db << "PRAGMA cache_size=-" + std::to_string(5 * 1024); // in -KB
db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit
db << "PRAGMA journal_mode=WAL"; // PRAGMA wal_autocheckpoint=10000; db << "PRAGMA journal_mode=MEMORY";
db << "PRAGMA temp_store=MEMORY";
db << "PRAGMA case_sensitive_like=true"; db << "PRAGMA case_sensitive_like=true";
if (fWipe) { if (fWipe) {
@ -93,7 +94,10 @@ CClaimTrie::CClaimTrie(bool fWipe, int height, int proportionalDelayFactor, std:
} }
CClaimTrieCacheBase::~CClaimTrieCacheBase() { CClaimTrieCacheBase::~CClaimTrieCacheBase() {
db << "rollback"; if (transacting) {
db << "rollback";
transacting = false;
}
} }
bool CClaimTrie::SyncToDisk() bool CClaimTrie::SyncToDisk()
@ -143,8 +147,8 @@ supportEntryType CClaimTrieCacheBase::getSupportsForName(const std::string& name
bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const
{ {
auto query = db << "SELECT validHeight FROM claims WHERE nodeName = ? AND txID = ? AND txN = ? " auto query = db << "SELECT validHeight FROM claims WHERE nodeName = ? AND txID = ? AND txN = ? "
"AND validHeight >= ? AND expirationHeight >= ? LIMIT 1" "AND validHeight >= ? AND expirationHeight > validHeight LIMIT 1"
<< name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight; << name << outPoint.hash << outPoint.n << nNextHeight;
for (auto&& row: query) { for (auto&& row: query) {
row >> nValidAtHeight; row >> nValidAtHeight;
return true; return true;
@ -155,8 +159,8 @@ bool CClaimTrieCacheBase::haveClaimInQueue(const std::string& name, const COutPo
bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const
{ {
auto query = db << "SELECT validHeight FROM supports WHERE nodeName = ? AND txID = ? AND txN = ? " auto query = db << "SELECT validHeight FROM supports WHERE nodeName = ? AND txID = ? AND txN = ? "
"AND validHeight >= ? AND expirationHeight >= ? LIMIT 1" "AND validHeight >= ? AND expirationHeight > validHeight LIMIT 1"
<< name << outPoint.hash << outPoint.n << nNextHeight << nNextHeight; << name << outPoint.hash << outPoint.n << nNextHeight;
for (auto&& row: query) { for (auto&& row: query) {
row >> nValidAtHeight; row >> nValidAtHeight;
return true; return true;
@ -202,6 +206,8 @@ bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::str
} }
void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() { void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() {
if (!transacting) return;
// your children are your nodes that match your key, go at least one longer, // your children are your nodes that match your key, go at least one longer,
// and have nothing in common with the other nodes in that set -- a hard query w/o parent // and have nothing in common with the other nodes in that set -- a hard query w/o parent
@ -310,7 +316,7 @@ std::size_t CClaimTrieCacheBase::getTotalClaimsInTrie() const
CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) const CAmount CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly) const
{ {
CAmount ret = 0; CAmount ret = 0;
std::string query("SELECT c.amount + SUM(SELECT s.amount FROM supports s " std::string query("SELECT (SELECT TOTAL(s.amount)+c.amount FROM supports s "
"WHERE s.supportedClaimID = c.claimID AND s.validHeight < ? AND s.expirationHeight >= ?) " "WHERE s.supportedClaimID = c.claimID AND s.validHeight < ? AND s.expirationHeight >= ?) "
"FROM claims c WHERE c.validHeight < ? AND s.expirationHeight >= ?"); "FROM claims c WHERE c.validHeight < ? AND s.expirationHeight >= ?");
if (fControllingOnly) if (fControllingOnly)
@ -433,14 +439,16 @@ bool CClaimTrieCacheBase::checkConsistency()
bool CClaimTrieCacheBase::flush() bool CClaimTrieCacheBase::flush()
{ {
getMerkleHash(); if (transacting) {
try { getMerkleHash();
db << "commit"; try {
db << "begin"; db << "commit";
} }
catch (const std::exception& e) { catch (const std::exception& e) {
LogPrintf("ERROR in ClaimTrieCache flush: %s\n", e.what()); LogPrintf("ERROR in ClaimTrieCache flush: %s\n", e.what());
return false; return false;
}
transacting = false;
} }
base->nNextHeight = nNextHeight; base->nNextHeight = nNextHeight;
return true; return true;
@ -475,11 +483,16 @@ bool CClaimTrieCacheBase::ValidateTipMatches(const CBlockIndex* tip)
} }
CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
: base(base), db(base->dbPath, sharedConfig) : base(base), db(base->dbPath, sharedConfig), transacting(false)
{ {
assert(base); assert(base);
nNextHeight = base->nNextHeight; nNextHeight = base->nNextHeight;
db << "begin";
db << "PRAGMA cache_size=-" + std::to_string(200 * 1024); // in -KB
db << "PRAGMA synchronous=NORMAL"; // don't disk sync after transaction commit
db << "PRAGMA journal_mode=MEMORY";
db << "PRAGMA temp_store=MEMORY";
db << "PRAGMA case_sensitive_like=true";
} }
int CClaimTrieCacheBase::expirationTime() const int CClaimTrieCacheBase::expirationTime() const
@ -492,8 +505,10 @@ uint256 CClaimTrieCacheBase::getMerkleHash()
ensureTreeStructureIsUpToDate(); ensureTreeStructureIsUpToDate();
std::unique_ptr<uint256> hash; std::unique_ptr<uint256> hash;
db << "SELECT hash FROM nodes WHERE name = ''" >> hash; db << "SELECT hash FROM nodes WHERE name = ''" >> hash;
if (hash == nullptr || hash->IsNull()) if (hash == nullptr || hash->IsNull()) {
assert(transacting); // no data changed but we didn't have the root hash there already?
return recursiveComputeMerkleHash("", false); return recursiveComputeMerkleHash("", false);
}
return *hash; return *hash;
} }
@ -510,6 +525,8 @@ bool CClaimTrieCacheBase::getLastTakeoverForName(const std::string& name, uint16
bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId,
CAmount nAmount, int nHeight, const std::vector<unsigned char>& metadata) CAmount nAmount, int nHeight, const std::vector<unsigned char>& metadata)
{ {
if (!transacting) { transacting = true; db << "begin"; }
auto delay = getDelayForName(name, claimId); auto delay = getDelayForName(name, claimId);
auto nodeName = adjustNameForValidHeight(name, nHeight + delay); auto nodeName = adjustNameForValidHeight(name, nHeight + delay);
auto expires = expirationTime() + nHeight; auto expires = expirationTime() + nHeight;
@ -527,6 +544,8 @@ bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& out
bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& outPoint, CAmount nAmount, bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& outPoint, CAmount nAmount,
const uint160& supportedClaimId, int nHeight, const std::vector<unsigned char>& metadata) const uint160& supportedClaimId, int nHeight, const std::vector<unsigned char>& metadata)
{ {
if (!transacting) { transacting = true; db << "begin"; }
auto delay = getDelayForName(name, supportedClaimId); auto delay = getDelayForName(name, supportedClaimId);
auto nodeName = adjustNameForValidHeight(name, nHeight + delay); auto nodeName = adjustNameForValidHeight(name, nHeight + delay);
auto expires = expirationTime() + nHeight; auto expires = expirationTime() + nHeight;
@ -541,6 +560,8 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o
bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, std::string& nodeName, int& validHeight) bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, std::string& nodeName, int& validHeight)
{ {
if (!transacting) { transacting = true; db << "begin"; }
auto query = db << "SELECT nodeName, validHeight FROM claims WHERE claimID = ?" auto query = db << "SELECT nodeName, validHeight FROM claims WHERE claimID = ?"
<< claimId; << claimId;
auto it = query.begin(); auto it = query.begin();
@ -553,6 +574,8 @@ bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, std::string& nodeN
bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight) bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight)
{ {
if (!transacting) { transacting = true; db << "begin"; }
auto query = db << "SELECT nodeName, validHeight FROM supports WHERE txID = ? AND txN = ?" auto query = db << "SELECT nodeName, validHeight FROM supports WHERE txID = ? AND txN = ?"
<< outPoint.hash << outPoint.n; << outPoint.hash << outPoint.n;
auto it = query.begin(); auto it = query.begin();
@ -857,6 +880,7 @@ bool CClaimTrieCacheBase::incrementBlock(insertUndoType& insertUndo, claimQueueR
// for every claim and support that becomes active this block set its node hash to null (aka, dirty) // for every claim and support that becomes active this block set its node hash to null (aka, dirty)
// for every claim and support that expires this block set its node hash to null and add it to the expire(Support)Undo // for every claim and support that expires this block set its node hash to null and add it to the expire(Support)Undo
// for all dirty nodes look for new takeovers // for all dirty nodes look for new takeovers
if (!transacting) { transacting = true; db << "begin"; }
{ {
db << "UPDATE nodes SET hash = NULL WHERE name IN " db << "UPDATE nodes SET hash = NULL WHERE name IN "
@ -955,6 +979,8 @@ void CClaimTrieCacheBase::activateAllFor(insertUndoType& insertUndo, insertUndoT
bool CClaimTrieCacheBase::decrementBlock(insertUndoType& insertUndo, claimQueueRowType& expireUndo, insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo) bool CClaimTrieCacheBase::decrementBlock(insertUndoType& insertUndo, claimQueueRowType& expireUndo, insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo)
{ {
if (!transacting) { transacting = true; db << "begin"; }
nNextHeight--; nNextHeight--;
for (auto it = expireSupportUndo.crbegin(); it != expireSupportUndo.crend(); ++it) { for (auto it = expireSupportUndo.crbegin(); it != expireSupportUndo.crend(); ++it) {

3
src/claimtrie.h
View file

@ -303,7 +303,7 @@ public:
CClaimTrie() = delete; CClaimTrie() = delete;
CClaimTrie(CClaimTrie&&) = delete; CClaimTrie(CClaimTrie&&) = delete;
CClaimTrie(const CClaimTrie&) = delete; CClaimTrie(const CClaimTrie&) = delete;
CClaimTrie(bool fWipe, int height, int proportionalDelayFactor = 32, std::size_t cacheMB=50); CClaimTrie(bool fWipe, int height, int proportionalDelayFactor = 32);
CClaimTrie& operator=(CClaimTrie&&) = delete; CClaimTrie& operator=(CClaimTrie&&) = delete;
CClaimTrie& operator=(const CClaimTrie&) = delete; CClaimTrie& operator=(const CClaimTrie&) = delete;
@ -409,6 +409,7 @@ protected:
CClaimTrie* base; CClaimTrie* base;
mutable sqlite::database db; mutable sqlite::database db;
int nNextHeight; // Height of the block that is being worked on, which is int nNextHeight; // Height of the block that is being worked on, which is
bool transacting;
// one greater than the height of the chain's tip // one greater than the height of the chain's tip
virtual uint256 recursiveComputeMerkleHash(const std::string& name, bool checkOnly); virtual uint256 recursiveComputeMerkleHash(const std::string& name, bool checkOnly);

8
src/claimtrieforks.cpp
View file

@ -62,6 +62,8 @@ bool CClaimTrieCacheExpirationFork::finalizeDecrement()
bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment) bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
{ {
if (!transacting) { transacting = true; db << "begin"; }
/* /*
If increment is True, we have forked to extend the expiration time, thus items in the expiration queue If increment is True, we have forked to extend the expiration time, thus items in the expiration queue
will have their expiration extended by "new expiration time - original expiration time" will have their expiration extended by "new expiration time - original expiration time"
@ -129,6 +131,8 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(bool f
if (nNextHeight != Params().GetConsensus().nNormalizedNameForkHeight) if (nNextHeight != Params().GetConsensus().nNormalizedNameForkHeight)
return false; return false;
if (!transacting) { transacting = true; db << "begin"; }
// run the one-time upgrade of all names that need to change // run the one-time upgrade of all names that need to change
// it modifies the (cache) trie as it goes, so we need to grab everything to be modified first // it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
@ -213,8 +217,8 @@ uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(const std::string& n
childHashes.push_back(*hash); childHashes.push_back(*hash);
} }
auto claimQuery = db << "SELECT c.txID, c.txN, c.validHeight, c.amount + " auto claimQuery = db << "SELECT c.txID, c.txN, c.validHeight, "
"SUM(SELECT s.amount FROM supports s WHERE s.supportedClaimID = c.claimID " "(SELECT TOTAL(s.amount)+c.amount FROM supports s WHERE s.supportedClaimID = c.claimID "
"AND s.validHeight < ? AND s.expirationHeight >= ?) as effectiveAmount" "AND s.validHeight < ? AND s.expirationHeight >= ?) as effectiveAmount"
"FROM claims c WHERE c.nodeName = ? AND c.validHeight < ? AND c.expirationHeight >= ? " "FROM claims c WHERE c.nodeName = ? AND c.validHeight < ? AND c.expirationHeight >= ? "
"ORDER BY effectiveAmount DESC, c.blockHeight, c.txID, c.txN" << nNextHeight << nNextHeight << name << nNextHeight << nNextHeight; "ORDER BY effectiveAmount DESC, c.blockHeight, c.txID, c.txN" << nNextHeight << nNextHeight << name << nNextHeight << nNextHeight;

21
src/init.cpp
View file

@ -371,7 +371,6 @@ void SetupServerArgs()
gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS);
gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS); gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS);
gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS); gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-claimtriecache=<n>", strprintf("Set claim trie cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS); gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS); gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS);
gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS);
@ -399,8 +398,6 @@ void SetupServerArgs()
hidden_args.emplace_back("-sysperms"); hidden_args.emplace_back("-sysperms");
#endif #endif
gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS); gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-memfile=<GiB>", "Use a memory mapped file for the claimtrie allocations (default: use RAM instead)", false, OptionsCategory::OPTIONS);
gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION); gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION); gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
gArgs.AddArg("-bantime=<n>", strprintf("Number of seconds to keep misbehaving peers from reconnecting (default: %u)", DEFAULT_MISBEHAVING_BANTIME), false, OptionsCategory::CONNECTION); gArgs.AddArg("-bantime=<n>", strprintf("Number of seconds to keep misbehaving peers from reconnecting (default: %u)", DEFAULT_MISBEHAVING_BANTIME), false, OptionsCategory::CONNECTION);
@ -1437,8 +1434,6 @@ bool AppInitMain()
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
g_memfileSize = gArgs.GetArg("-memfile", 0u);
bool fLoaded = false; bool fLoaded = false;
while (!fLoaded && !ShutdownRequested()) { while (!fLoaded && !ShutdownRequested()) {
bool fReset = fReindex; bool fReset = fReindex;
@ -1460,10 +1455,7 @@ bool AppInitMain()
pblocktree.reset(); pblocktree.reset();
pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset));
delete pclaimTrie; delete pclaimTrie;
int64_t trieCacheMB = gArgs.GetArg("-claimtriecache", nDefaultDbCache); pclaimTrie = new CClaimTrie(fReindex || fReindexChainState, 0);
trieCacheMB = std::min(trieCacheMB, nMaxDbCache);
trieCacheMB = std::max(trieCacheMB, nMinDbCache);
pclaimTrie = new CClaimTrie(fReindex || fReindexChainState, 32, trieCacheMB);
if (fReset) { if (fReset) {
pblocktree->WriteReindexing(true); pblocktree->WriteReindexing(true);
@ -1536,14 +1528,13 @@ bool AppInitMain()
} }
assert(chainActive.Tip() != nullptr); assert(chainActive.Tip() != nullptr);
} }
CClaimTrieCache trieCache(pclaimTrie);
if (!trieCache.ValidateTipMatches(chainActive.Tip()))
{ {
strLoadError = _("Error loading the claim trie from disk"); CClaimTrieCache trieCache(pclaimTrie);
break; if (!trieCache.ValidateTipMatches(chainActive.Tip())) {
strLoadError = _("Error loading the claim trie from disk");
break;
}
} }
if (!fReset) { if (!fReset) {
// Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate.
// It both disconnects blocks based on chainActive, and drops block data in // It both disconnects blocks based on chainActive, and drops block data in

9
src/miner.cpp
View file

@ -200,10 +200,11 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc
pblock->nNonce = 0; pblock->nNonce = 0;
pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]);
CClaimTrieCache trieCache(pclaimTrie); {
blockToCache(pblock, trieCache, nHeight); CClaimTrieCache trieCache(pclaimTrie);
pblock->hashClaimTrie = trieCache.getMerkleHash(); blockToCache(pblock, trieCache, nHeight);
pblock->hashClaimTrie = trieCache.getMerkleHash();
}
CValidationState state; CValidationState state;
if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) {
throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state)));

199
src/test/claimtriecache_tests.cpp
View file

@ -17,26 +17,47 @@ public:
{ {
} }
bool insertClaimIntoTrie(const std::string& key, const CClaimValue& data) bool insertClaimIntoTrie(const std::string& key, const CClaimValue& value)
{ {
return addClaim(key, data.outPoint, data.claimId, data.nAmount, data.nHeight, {}); auto p = value.outPoint;
if (p.hash.IsNull())
p.hash = Hash(key.begin(), key.end());
auto c = value.claimId;
if (c.IsNull())
c = ClaimIdHash(p.hash, p.n);
return addClaim(key, p, c, value.nAmount, value.nHeight, {});
} }
bool removeClaimFromTrie(const std::string& key, const COutPoint& p) { bool removeClaimFromTrie(const std::string& key, const COutPoint& outPoint) {
int validHeight; int validHeight;
std::string nodeName; std::string nodeName;
auto p = outPoint;
if (p.hash.IsNull())
p.hash = Hash(key.begin(), key.end());
auto ret = removeClaim(ClaimIdHash(p.hash, p.n), nodeName, validHeight); auto ret = removeClaim(ClaimIdHash(p.hash, p.n), nodeName, validHeight);
assert(!ret || nodeName == key); assert(!ret || nodeName == key);
return ret; return ret;
} }
bool insertSupportIntoMap(const std::string& key, const CSupportValue& value) { bool insertSupportIntoMap(const std::string& key, const CSupportValue& value) {
return addSupport(key, value.outPoint, value.nAmount, value.supportedClaimId, value.nHeight, {}); auto p = value.outPoint;
if (p.hash.IsNull())
p.hash = Hash(key.begin(), key.end());
return addSupport(key, p, value.nAmount, value.supportedClaimId, value.nHeight, {});
} }
bool removeSupportFromMap(const std::string& key, const COutPoint& p) { bool removeSupportFromMap(const std::string& key, const COutPoint& outPoint) {
int validHeight; int validHeight;
std::string nodeName; std::string nodeName;
auto p = outPoint;
if (p.hash.IsNull())
p.hash = Hash(key.begin(), key.end());
auto ret = removeSupport(p, nodeName, validHeight); auto ret = removeSupport(p, nodeName, validHeight);
assert(!ret || nodeName == key); assert(!ret || nodeName == key);
return ret; return ret;
@ -60,7 +81,6 @@ BOOST_AUTO_TEST_CASE(merkle_hash_single_test)
BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test) BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
{ {
uint256 hash0(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); uint256 hash0(uint256S("0000000000000000000000000000000000000000000000000000000000000001"));
uint160 hash160;
CMutableTransaction tx1 = BuildTransaction(hash0); CMutableTransaction tx1 = BuildTransaction(hash0);
COutPoint tx1OutPoint(tx1.GetHash(), 0); COutPoint tx1OutPoint(tx1.GetHash(), 0);
CMutableTransaction tx2 = BuildTransaction(tx1.GetHash()); CMutableTransaction tx2 = BuildTransaction(tx1.GetHash());
@ -88,92 +108,102 @@ BOOST_AUTO_TEST_CASE(merkle_hash_multiple_test)
BOOST_CHECK(pclaimTrie->empty()); BOOST_CHECK(pclaimTrie->empty());
CClaimTrieCacheTest ntState(pclaimTrie); {
ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, hash160, 50, 100, 200)); CClaimTrieCacheTest ntState(pclaimTrie);
ntState.insertClaimIntoTrie(std::string("test2"), CClaimValue(tx2OutPoint, hash160, 50, 100, 200)); ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, {}, 50, 0, 0));
ntState.insertClaimIntoTrie(std::string("test2"), CClaimValue(tx2OutPoint, {}, 50, 0, 0));
BOOST_CHECK(pclaimTrie->empty()); BOOST_CHECK(pclaimTrie->empty());
BOOST_CHECK(ntState.getTotalClaimsInTrie() > 0); BOOST_CHECK_EQUAL(ntState.getTotalClaimsInTrie(), 2U);
BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash1); BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash1);
ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, hash160, 50, 101, 201)); ntState.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, {}, 50, 1, 0));
BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash1); BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash1);
ntState.insertClaimIntoTrie(std::string("tes"), CClaimValue(tx4OutPoint, hash160, 50, 100, 200)); ntState.insertClaimIntoTrie(std::string("tes"), CClaimValue(tx4OutPoint, {}, 50, 0, 0));
BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2); BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
ntState.insertClaimIntoTrie(std::string("testtesttesttest"), CClaimValue(tx5OutPoint, hash160, 50, 100, 200)); ntState.insertClaimIntoTrie(std::string("testtesttesttest"),
ntState.removeClaimFromTrie(std::string("testtesttesttest"), tx5OutPoint); CClaimValue(tx5OutPoint, {}, 50, 0, 0));
BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2); ntState.removeClaimFromTrie(std::string("testtesttesttest"), tx5OutPoint);
ntState.flush(); BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
ntState.flush();
BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2); BOOST_CHECK_EQUAL(ntState.getMerkleHash(), hash2);
BOOST_CHECK(ntState.checkConsistency()); BOOST_CHECK(ntState.checkConsistency());
}
{
CClaimTrieCacheTest ntState1(pclaimTrie);
ntState1.removeClaimFromTrie(std::string("test"), tx1OutPoint);
ntState1.removeClaimFromTrie(std::string("test2"), tx2OutPoint);
ntState1.removeClaimFromTrie(std::string("test"), tx3OutPoint);
ntState1.removeClaimFromTrie(std::string("tes"), tx4OutPoint);
CClaimTrieCacheTest ntState1(pclaimTrie); BOOST_CHECK_EQUAL(ntState1.getMerkleHash(), hash0);
ntState1.removeClaimFromTrie(std::string("test"), tx1OutPoint); }
ntState1.removeClaimFromTrie(std::string("test2"), tx2OutPoint); {
ntState1.removeClaimFromTrie(std::string("test"), tx3OutPoint); CClaimTrieCacheTest ntState2(pclaimTrie);
ntState1.removeClaimFromTrie(std::string("tes"), tx4OutPoint); ntState2.insertClaimIntoTrie(std::string("abab"), CClaimValue(tx6OutPoint, {}, 50, 0, 0));
ntState2.removeClaimFromTrie(std::string("test"), tx1OutPoint);
BOOST_CHECK_EQUAL(ntState1.getMerkleHash(), hash0); BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3);
CClaimTrieCacheTest ntState2(pclaimTrie); ntState2.flush();
ntState2.insertClaimIntoTrie(std::string("abab"), CClaimValue(tx6OutPoint, hash160, 50, 100, 200));
ntState2.removeClaimFromTrie(std::string("test"), tx1OutPoint);
BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3); BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3);
BOOST_CHECK(ntState2.checkConsistency());
}
{
CClaimTrieCacheTest ntState3(pclaimTrie);
ntState3.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, {}, 50, 0, 0));
BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4);
ntState3.flush();
BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4);
BOOST_CHECK(ntState3.checkConsistency());
}
{
CClaimTrieCacheTest ntState4(pclaimTrie);
ntState4.removeClaimFromTrie(std::string("abab"), tx6OutPoint);
BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
ntState4.flush();
BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
BOOST_CHECK(ntState4.checkConsistency());
}
{
CClaimTrieCacheTest ntState5(pclaimTrie);
ntState5.removeClaimFromTrie(std::string("test"), tx3OutPoint);
ntState2.flush(); BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
ntState5.flush();
BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
BOOST_CHECK(ntState5.checkConsistency());
}
{
CClaimTrieCacheTest ntState6(pclaimTrie);
ntState6.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, {}, 50, 0, 0));
BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
BOOST_CHECK_EQUAL(ntState2.getMerkleHash(), hash3); ntState6.flush();
BOOST_CHECK(ntState2.checkConsistency()); BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
BOOST_CHECK(ntState6.checkConsistency());
}
{
CClaimTrieCacheTest ntState7(pclaimTrie);
ntState7.removeClaimFromTrie(std::string("test"), tx3OutPoint);
ntState7.removeClaimFromTrie(std::string("test"), tx1OutPoint);
ntState7.removeClaimFromTrie(std::string("tes"), tx4OutPoint);
ntState7.removeClaimFromTrie(std::string("test2"), tx2OutPoint);
CClaimTrieCacheTest ntState3(pclaimTrie); BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
ntState3.insertClaimIntoTrie(std::string("test"), CClaimValue(tx1OutPoint, hash160, 50, 100, 200)); ntState7.flush();
BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4); BOOST_CHECK(pclaimTrie->empty());
ntState3.flush(); BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
BOOST_CHECK(!pclaimTrie->empty()); BOOST_CHECK(ntState7.checkConsistency());
BOOST_CHECK_EQUAL(ntState3.getMerkleHash(), hash4); }
BOOST_CHECK(ntState3.checkConsistency());
CClaimTrieCacheTest ntState4(pclaimTrie);
ntState4.removeClaimFromTrie(std::string("abab"), tx6OutPoint);
BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
ntState4.flush();
BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState4.getMerkleHash(), hash2);
BOOST_CHECK(ntState4.checkConsistency());
CClaimTrieCacheTest ntState5(pclaimTrie);
ntState5.removeClaimFromTrie(std::string("test"), tx3OutPoint);
BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
ntState5.flush();
BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState5.getMerkleHash(), hash2);
BOOST_CHECK(ntState5.checkConsistency());
CClaimTrieCacheTest ntState6(pclaimTrie);
ntState6.insertClaimIntoTrie(std::string("test"), CClaimValue(tx3OutPoint, hash160, 50, 101, 201));
BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
ntState6.flush();
BOOST_CHECK(!pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState6.getMerkleHash(), hash2);
BOOST_CHECK(ntState6.checkConsistency());
CClaimTrieCacheTest ntState7(pclaimTrie);
ntState7.removeClaimFromTrie(std::string("test"), tx3OutPoint);
ntState7.removeClaimFromTrie(std::string("test"), tx1OutPoint);
ntState7.removeClaimFromTrie(std::string("tes"), tx4OutPoint);
ntState7.removeClaimFromTrie(std::string("test2"), tx2OutPoint);
BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
ntState7.flush();
BOOST_CHECK(pclaimTrie->empty());
BOOST_CHECK_EQUAL(ntState7.getMerkleHash(), hash0);
BOOST_CHECK(ntState7.checkConsistency());
} }
BOOST_AUTO_TEST_CASE(basic_insertion_info_test) BOOST_AUTO_TEST_CASE(basic_insertion_info_test)
@ -273,7 +303,7 @@ BOOST_AUTO_TEST_CASE(trie_stays_consistent_test)
"goodness", "goodnight", "goodnatured", "goods", "go", "goody", "goo" "goodness", "goodnight", "goodnatured", "goods", "go", "goody", "goo"
}; };
CClaimTrie trie(false, 1); CClaimTrie trie(false, 0, 1);
CClaimTrieCacheTest cache(&trie); CClaimTrieCacheTest cache(&trie);
CClaimValue value; CClaimValue value;
@ -304,7 +334,7 @@ BOOST_AUTO_TEST_CASE(takeover_workaround_triggers)
BOOST_SCOPE_EXIT(&consensus, currentMax) { consensus.nMaxTakeoverWorkaroundHeight = currentMax; } BOOST_SCOPE_EXIT(&consensus, currentMax) { consensus.nMaxTakeoverWorkaroundHeight = currentMax; }
BOOST_SCOPE_EXIT_END BOOST_SCOPE_EXIT_END
CClaimTrie trie(false, 1); CClaimTrie trie(false, 0, 1);
CClaimTrieCacheTest cache(&trie); CClaimTrieCacheTest cache(&trie);
insertUndoType icu, isu; claimQueueRowType ecu; supportQueueRowType esu; insertUndoType icu, isu; claimQueueRowType ecu; supportQueueRowType esu;
@ -324,7 +354,6 @@ BOOST_AUTO_TEST_CASE(takeover_workaround_triggers)
BOOST_CHECK(cache.incrementBlock(icu, ecu, isu, esu)); BOOST_CHECK(cache.incrementBlock(icu, ecu, isu, esu));
BOOST_CHECK(cache.flush()); BOOST_CHECK(cache.flush());
BOOST_CHECK(cache.incrementBlock(icu, ecu, isu, esu)); BOOST_CHECK(cache.incrementBlock(icu, ecu, isu, esu));
BOOST_CHECK_EQUAL(0, cache.getTotalNamesInTrie());
CSupportValue temp; CSupportValue temp;
CClaimValue cv; CClaimValue cv;

2
src/test/test_bitcoin.cpp
View file

@ -138,7 +138,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha
pblocktree.reset(new CBlockTreeDB(1 << 20, true)); pblocktree.reset(new CBlockTreeDB(1 << 20, true));
pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true));
pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get()));
pclaimTrie = new CClaimTrie(true, 1); pclaimTrie = new CClaimTrie(true, 0, 1);
if (!LoadGenesisBlock(chainparams)) { if (!LoadGenesisBlock(chainparams)) {
throw std::runtime_error("LoadGenesisBlock failed."); throw std::runtime_error("LoadGenesisBlock failed.");
} }