LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

added a test for putting a lot of data into the claimtrie

Browse source 
updated to dodge expiration fork
This commit is contained in:
Brannon King 2019-07-01 12:40:18 -06:00
parent 24269e177f
commit 8d955fdd22
5 changed files with 141 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/chainparams.cpp
View file

@ -357,7 +357,7 @@ public:
consensus.nPowTargetSpacing = 1;
consensus.nOriginalClaimExpirationTime = 500;
consensus.nExtendedClaimExpirationTime = 600;
consensus.nExtendedClaimExpirationForkHeight = 800;
consensus.nExtendedClaimExpirationForkHeight = 8000;
consensus.nAllowMinDiffMinHeight = -1;
consensus.nAllowMinDiffMaxHeight = -1;
consensus.nNormalizedNameForkHeight = 250; // SDK depends upon this number

8
src/claimtrie.cpp
View file

@ -1518,9 +1518,9 @@ void CClaimTrieCacheBase::addToExpirationQueue(int nExpirationHeight, nameOutPoi
void CClaimTrieCacheBase::removeFromExpirationQueue(const std::string& name, const COutPoint& outPoint, int expirationHeight) const
{
expirationQueueType::iterator itQueueRow = getExpirationQueueCacheRow(expirationHeight, false);
expirationQueueRowType::iterator itQueue;
if (itQueueRow != expirationQueueCache.end())
{
expirationQueueRowType::iterator itQueue;
for (itQueue = itQueueRow->second.begin(); itQueue != itQueueRow->second.end(); ++itQueue)
{
if (outPoint == itQueue->outPoint && name == itQueue->name)
@ -1822,17 +1822,17 @@ void CClaimTrieCacheBase::addSupportToExpirationQueue(int nExpirationHeight, nam
void CClaimTrieCacheBase::removeSupportFromExpirationQueue(const std::string& name, const COutPoint& outPoint, int expirationHeight) const
{
expirationQueueType::iterator itQueueRow = getSupportExpirationQueueCacheRow(expirationHeight, false);
expirationQueueRowType::iterator itQueue;
if (itQueueRow != supportExpirationQueueCache.end())
{
expirationQueueRowType::iterator itQueue;
for (itQueue = itQueueRow->second.begin(); itQueue != itQueueRow->second.end(); ++itQueue)
{
if (outPoint == itQueue->outPoint && name == itQueue->name)
break;
}
if (itQueue != itQueueRow->second.end())
itQueueRow->second.erase(itQueue);
}
if (itQueue != itQueueRow->second.end())
itQueueRow->second.erase(itQueue);
}
expirationQueueType::iterator CClaimTrieCacheBase::getSupportExpirationQueueCacheRow(int nHeight, bool createIfNotExists) const

4
src/memusage.h
View file

@ -159,8 +159,8 @@ static inline size_t DynamicUsage(const std::unordered_set<X, Y>& s)
return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count());
}
template<typename X, typename Y, typename Z>
static inline size_t DynamicUsage(const std::unordered_map<X, Y, Z>& m)
template<typename X, typename Y, typename ... Z>
static inline size_t DynamicUsage(const std::unordered_map<X, Y, Z...>& m)
{
return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count());
}

11
src/rpc/claimtrie.cpp
View file

@ -462,15 +462,15 @@ UniValue getclaimsforname(const JSONRPCRequest& request)
claimSupportMapType claimSupportMap;
UniValue unmatchedSupports(UniValue::VARR);
for (std::vector<CClaimValue>::const_iterator itClaims = claimsForName.claims.begin(); itClaims != claimsForName.claims.end(); ++itClaims)
for (auto itClaims = claimsForName.claims.begin(); itClaims != claimsForName.claims.end(); ++itClaims)
{
claimAndSupportsType claimAndSupports = std::make_pair(*itClaims, std::vector<CSupportValue>());
claimSupportMap.insert(std::pair<uint160, claimAndSupportsType>(itClaims->claimId, claimAndSupports));
claimSupportMap.emplace(itClaims->claimId, claimAndSupports);
}
for (std::vector<CSupportValue>::const_iterator itSupports = claimsForName.supports.begin(); itSupports != claimsForName.supports.end(); ++itSupports)
for (auto itSupports = claimsForName.supports.begin(); itSupports != claimsForName.supports.end(); ++itSupports)
{
claimSupportMapType::iterator itClaimAndSupports = claimSupportMap.find(itSupports->supportedClaimId);
auto itClaimAndSupports = claimSupportMap.find(itSupports->supportedClaimId);
if (itClaimAndSupports == claimSupportMap.end())
unmatchedSupports.push_back(supportToJSON(coinsCache, *itSupports));
else
@ -481,8 +481,9 @@ UniValue getclaimsforname(const JSONRPCRequest& request)
result.pushKV("nLastTakeoverHeight", claimsForName.nLastTakeoverHeight);
result.pushKV("normalized_name", claimsForName.name);
for (claimSupportMapType::const_iterator itClaimsAndSupports = claimSupportMap.begin(); itClaimsAndSupports != claimSupportMap.end(); ++itClaimsAndSupports)
for (auto itClaims = claimsForName.claims.begin(); itClaims != claimsForName.claims.end(); ++itClaims)
{
auto itClaimsAndSupports = claimSupportMap.find(itClaims->claimId);
const auto nEffectiveAmount = trieCache.getEffectiveAmountForClaim(claimsForName, itClaimsAndSupports->first);
UniValue claimObj = claimAndSupportsToJSON(coinsCache, nEffectiveAmount, itClaimsAndSupports);
claimObjs.push_back(claimObj);

162
src/test/claimtriebranching_tests.cpp
View file

@ -20,8 +20,10 @@
#include <boost/test/unit_test.hpp>
#include <iostream>
#include <random>
extern ::CChainState g_chainstate;
extern ::ArgsManager gArgs;
using namespace std;
@ -125,7 +127,6 @@ struct ClaimTrieChainFixture
int coinbase_txs_used;
int unique_block_counter;
int normalization_original;
unsigned int num_txs;
unsigned int num_txs_for_next_block;
int64_t expirationForkHeight;
@ -139,8 +140,12 @@ struct ClaimTrieChainFixture
BOOST_CHECK_EQUAL(pclaimTrie->nCurrentHeight, chainActive.Height() + 1);
setNormalizationForkHeight(1000000);
gArgs.ForceSetArg("-limitancestorcount", "1000000");
gArgs.ForceSetArg("-limitancestorsize", "1000000");
gArgs.ForceSetArg("-limitdescendantcount", "1000000");
gArgs.ForceSetArg("-limitdescendantsize", "1000000");
num_txs_for_next_block = 0;
num_txs = 0;
coinbase_txs_used = 0;
unique_block_counter = 0;
// generate coinbases to spend
@ -225,12 +230,6 @@ struct ClaimTrieChainFixture
void CommitTx(const CMutableTransaction &tx) {
num_txs_for_next_block++;
num_txs++;
if(num_txs > coinbase_txs.size())
{
//ran out of coinbases to spend
assert(0);
}
/* TestMemPoolEntryHelper entry; */
/* LOCK(mempool.cs); */
@ -238,16 +237,15 @@ struct ClaimTrieChainFixture
CValidationState state;
CAmount txFeeRate = CAmount(0);
LOCK(cs_main);
BOOST_CHECK_EQUAL(AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), nullptr, nullptr, false, txFeeRate, false), true);
assert(AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), nullptr, nullptr, false, txFeeRate, false));
}
//spend a bid into some non claimtrie related unspent
CMutableTransaction Spend(const CTransaction &prev)
{
uint32_t prevout = 0;
CMutableTransaction tx = BuildTransaction(prev, prevout);
CMutableTransaction tx = BuildTransaction(prev, 0);
tx.vout[0].scriptPubKey = CScript() << OP_TRUE;
tx.vout[0].nValue = 1;
tx.vout[0].nValue = prev.vout[0].nValue;
CommitTx(tx);
return tx;
@ -256,10 +254,16 @@ struct ClaimTrieChainFixture
//make claim at the current block
CMutableTransaction MakeClaim(const CTransaction& prev, std::string name, std::string value, CAmount quantity)
{
uint32_t prevout = 0;
CMutableTransaction tx = BuildTransaction(prev, prevout);
uint32_t prevout = prev.vout.size() - 1;
while (prevout > 0 && prev.vout[prevout].nValue < quantity)
--prevout;
CMutableTransaction tx = BuildTransaction(prev, prevout, prev.vout[prevout].nValue > quantity ? 2 : 1);
tx.vout[0].scriptPubKey = ClaimNameScript(name, value);
tx.vout[0].nValue = quantity;
if (tx.vout.size() > 1) {
tx.vout[1].scriptPubKey = CScript() << OP_TRUE;
tx.vout[1].nValue = prev.vout[prevout].nValue - quantity;
}
CommitTx(tx);
return tx;
@ -273,10 +277,16 @@ struct ClaimTrieChainFixture
//make support at the current block
CMutableTransaction MakeSupport(const CTransaction &prev, const CTransaction &claimtx, std::string name, CAmount quantity)
{
uint32_t prevout = 0;
CMutableTransaction tx = BuildTransaction(prev,prevout);
uint32_t prevout = prev.vout.size() - 1;
while (prevout > 0 && prev.vout[prevout].nValue < quantity)
--prevout;
CMutableTransaction tx = BuildTransaction(prev, prevout, prev.vout[prevout].nValue > quantity ? 2 : 1);
tx.vout[0].scriptPubKey = SupportClaimScript(name, ClaimIdHash(claimtx.GetHash(), 0));
tx.vout[0].nValue = quantity;
if (tx.vout.size() > 1) {
tx.vout[1].scriptPubKey = CScript() << OP_TRUE;
tx.vout[1].nValue = prev.vout[prevout].nValue - quantity;
}
CommitTx(tx);
return tx;
@ -286,8 +296,7 @@ struct ClaimTrieChainFixture
CMutableTransaction MakeUpdate(const CTransaction &prev, std::string name, std::string value,
uint160 claimId, CAmount quantity)
{
uint32_t prevout = 0;
CMutableTransaction tx = BuildTransaction(prev, prevout);
CMutableTransaction tx = BuildTransaction(prev, 0);
tx.vout[0].scriptPubKey = UpdateClaimScript(name, claimId, value);
tx.vout[0].nValue = quantity;
@ -322,24 +331,14 @@ struct ClaimTrieChainFixture
//disconnect i blocks from tip
void DecrementBlocks(int num_blocks)
{
for (int i = 0; i < num_blocks; i++)
CValidationState state;
{
CValidationState state;
{
LOCK(cs_main);
CBlockIndex* pblockindex = chainActive.Tip();
BOOST_CHECK_EQUAL(InvalidateBlock(state, Params(), pblockindex), true);
}
if (state.IsValid())
{
BOOST_CHECK_EQUAL(ActivateBestChain(state, Params()), true);
}
else
{
BOOST_FAIL("removing block failed");
}
LOCK(cs_main);
CBlockIndex* pblockindex = chainActive[chainActive.Height() - num_blocks + 1];
BOOST_CHECK_EQUAL(InvalidateBlock(state, Params(), pblockindex), true);
}
BOOST_CHECK_EQUAL(state.IsValid(), true);
BOOST_CHECK_EQUAL(ActivateBestChain(state, Params()), true);
mempool.clear();
num_txs_for_next_block = 0;
}
@ -363,6 +362,101 @@ struct ClaimTrieChainFixture
}
};
std::vector<std::string> random_strings(std::size_t count)
{
static auto& chrs = "0123456789"
"abcdefghijklmnopqrstuvwxyz"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ";
std::mt19937 rg(42);
std::uniform_int_distribution<std::string::size_type> pick(1, sizeof(chrs) - 2);
std::unordered_set<std::string> strings;
strings.reserve(count);
while(strings.size() < count) {
auto length = pick(rg);
std::string s;
s.reserve(length);
while (length--)
s += chrs[pick(rg)];
strings.emplace(s);
}
std::vector<std::string> ret(strings.begin(), strings.end());
std::shuffle(ret.begin(), ret.end(), rg);
return ret;
}
BOOST_AUTO_TEST_CASE(triehash_fuzzer_test)
{
ClaimTrieChainFixture fixture;
auto envClaims = std::getenv("TRIEHASH_FUZZER_CLAIMS");
auto envBlocks = std::getenv("TRIEHASH_FUZZER_BLOCKS");
const int claimsPerBlock = envClaims ? atoi(envClaims) : 100;
const int blocks = envBlocks ? atoi(envBlocks) : 13;
auto names = random_strings(blocks * claimsPerBlock);
std::mt19937 rg(42);
std::uniform_int_distribution<int> pick4(0, 4);
std::uniform_int_distribution<std::size_t> pick;
std::unordered_map<std::string, std::vector<CMutableTransaction>> existingClaims;
std::vector<CMutableTransaction> existingSupports;
std::string value(1024, 'c');
std::vector<CTransaction> cb {fixture.GetCoinbase()};
for (int i = 0; i < blocks; ++i) {
for (int j = 0; j < claimsPerBlock; ++j) {
auto name = names[i * claimsPerBlock + j];
auto supportFront = pick4(rg) == 0;
auto supportBack = pick4(rg) == 0;
auto removeClaim = pick4(rg) == 0;
auto removeSupport = pick4(rg) == 0;
auto hit = existingClaims.find(name);
if (supportFront && hit != existingClaims.end() && hit->second.size()) {
auto tx = fixture.MakeSupport(cb.back(), hit->second[pick(rg) % hit->second.size()], name, 2);
existingSupports.push_back(tx);
cb.emplace_back(std::move(tx));
}
if (removeClaim && hit != existingClaims.end() && hit->second.size()) {
auto idx = pick(rg) % hit->second.size();
fixture.Spend(hit->second[idx]);
hit->second.erase(hit->second.begin() + idx);
}
else {
auto tx = fixture.MakeClaim(cb.back(), name, value, 2);
existingClaims[name].push_back(tx);
hit = existingClaims.find(name);
cb.emplace_back(std::move(tx));
}
if (supportBack && hit != existingClaims.end() && hit->second.size()) {
auto tx = fixture.MakeSupport(cb.back(), hit->second[pick(rg) % hit->second.size()], name, 2);
existingSupports.push_back(tx);
cb.emplace_back(std::move(tx));
}
if (removeSupport && (i & 7) == 7 && !existingSupports.empty()) {
const auto tidx = pick(rg) % existingSupports.size();
const auto tx = existingSupports[tidx];
fixture.Spend(tx);
existingSupports.erase(existingSupports.begin() + tidx);
}
if (cb.back().GetValueOut() < 10 || cb.size() > 40000) {
cb.clear();
cb.push_back(fixture.GetCoinbase());
}
}
std::cerr << "Block: " << i << std::endl;
fixture.IncrementBlocks(1);
}
std::cerr << "Hash: " << pclaimTrie->getMerkleHash().GetHex() << std::endl;
}
/*
claims