LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Merge branch 'add_maxblockfilesize_param'

Browse source
This commit is contained in:
Brannon King 2020-02-21 10:23:13 -07:00
commit a4f8b5c358
15 changed files with 53 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.travis.yml
View file

@ -71,7 +71,7 @@ jobs:
dist: xenial
language: minimal
git:
depth: 3
clone: false
install:
- mkdir -p testrun && cd testrun
- curl http://build.lbry.io/lbrycrd/${TRAVIS_BRANCH}/lbrycrd-${NAME}-test.zip -o temp.zip

3
src/init.cpp
View file

@ -370,6 +370,7 @@ void SetupServerArgs()
gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS);
gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS);
gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", false, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxblockfilesize=<n>", strprintf("Keep the block files below n megabytes each. (default: %d, minimum: 8)", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), false, OptionsCategory::OPTIONS);
@ -1427,7 +1428,7 @@ bool AppInitMain()
// however, we want the claimtrie cache to be larger than the others
int64_t nBlockTreeDBCache = std::min(nTotalCache / 4, nMaxBlockDBCache << 20);
int64_t nCoinDBCache = std::min(nTotalCache / 8, nMaxCoinsDBCache << 20);
int64_t nCoinDBCache = std::min(nTotalCache / 4, nMaxCoinsDBCache << 20);
int64_t nClaimtrieCache = nTotalCache / 4;
nTotalCache -= nBlockTreeDBCache;
nTotalCache -= nCoinDBCache;

3
src/policy/fees.cpp
View file

@ -288,7 +288,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal,
nConf += confAvg[periodTarget - 1][bucket];
totalNum += txCtAvg[bucket];
failNum += failAvg[periodTarget - 1][bucket];
for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++)
const auto maxConfirms = GetMaxConfirms();
for (unsigned int confct = confTarget; confct < maxConfirms; ++confct)
extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket];
extraNum += oldUnconfTxs[bucket];
// If we have enough transaction data points in this range of buckets,

9
src/rpc/mining.cpp
View file

@ -142,9 +142,10 @@ UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGen
continue;
}
std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock);
if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr))
auto lastInBatch = ++nHeight >= nHeightEnd;
if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr, lastInBatch))
throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted");
++nHeight;
blockHashes.push_back(pblock->GetHash().GetHex());
//mark script as important because it was used at least for one coinbase output if the script came from the wallet
@ -173,6 +174,10 @@ static UniValue generatetoaddress(const JSONRPCRequest& request)
+ HelpExampleCli("generatetoaddress", "11 \"myaddress\"")
);
if (IsInitialBlockDownload()) {
throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "generatetoaddress is not available during the initial block download");
}
int nGenerate = request.params[0].get_int();
uint64_t nMaxTries = 1000000;
if (!request.params[2].isNull()) {

2
src/test/claimtriefixture.cpp
View file

@ -192,7 +192,7 @@ bool ClaimTrieChainFixture::CreateBlock(const std::unique_ptr<CBlockTemplate>& p
break;
}
}
auto success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(*pblock), true, nullptr);
auto success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(*pblock), true, nullptr, false);
return success && pblock->GetHash() == chainActive.Tip()->GetBlockHash();
}

2
src/test/claimtriehashfork_tests.cpp
View file

@ -395,7 +395,7 @@ BOOST_AUTO_TEST_CASE(bogus_claimtrie_hash_test)
break;
}
}
bool success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(pblockTemp->block), true, nullptr);
bool success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(pblockTemp->block), true, nullptr, false);
// will process , but will not be connected
BOOST_CHECK(success);
BOOST_CHECK(pblockTemp->block.GetHash() != chainActive.Tip()->GetBlockHash());

2
src/test/miner_tests.cpp
View file

@ -254,7 +254,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
#endif
}
std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock);
BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr));
BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr, false));
}
LOCK(cs_main);

2
src/test/test_bitcoin.cpp
View file

@ -230,7 +230,7 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>&
while (!CheckProofOfWork(block.GetPoWHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce;
std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block);
ProcessNewBlock(chainparams, shared_pblock, true, nullptr);
ProcessNewBlock(chainparams, shared_pblock, true, nullptr, false);
CBlock result = block;
return result;

6
src/test/validation_block_tests.cpp
View file

@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering)
BOOST_CHECK(ProcessNewBlockHeaders(headers, state, Params()));
// Connect the genesis block and drain any outstanding events
ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored);
ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored, false);
SyncWithValidationInterfaceQueue();
// subscribe to events (this subscriber will validate event ordering)
@ -159,13 +159,13 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering)
bool ignored;
for (int i = 0; i < 1000; i++) {
auto block = blocks[GetRand(blocks.size() - 1)];
ProcessNewBlock(Params(), block, true, &ignored);
ProcessNewBlock(Params(), block, true, &ignored, false);
}
// to make sure that eventually we process the full chain - do it here
for (auto block : blocks) {
if (block->vtx.size() == 1) {
bool processed = ProcessNewBlock(Params(), block, true, &ignored);
bool processed = ProcessNewBlock(Params(), block, true, &ignored, false);
assert(processed);
}
}

3
src/txdb.cpp
View file

@ -256,7 +256,8 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const
CCoinsViewDBCursor::CCoinsViewDBCursor(const uint256 &hashBlockIn, const CCoinsViewDB* view)
: CCoinsViewCursor(hashBlockIn), owner(view),
query(owner->db << "SELECT txID, txN, isCoinbase, blockHeight, amount, script FROM unspent")
query(owner->db << "SELECT txID, txN, isCoinbase, blockHeight, amount, script "
"FROM unspent ORDER BY txID ASC, txN ASC")
{
iter = query.begin();
}

2
src/txdb.h
View file

@ -60,7 +60,7 @@ static const int64_t nMinDbCache = 4;
//! Max memory allocated to block tree DB specific cache
static const int64_t nMaxBlockDBCache = 260;
//! Max memory allocated to coin DB specific cache (MiB)
static const int64_t nMaxCoinsDBCache = 40;
static const int64_t nMaxCoinsDBCache = 200;
struct CDiskTxPos : public CDiskBlockPos

36
src/validation.cpp
View file

@ -2355,8 +2355,8 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha
assert(pindexDelete->pprev->hashClaimTrie == trieCache.getMerkleHash());
}
LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI);
// Write the chain state to disk, if necessary.
if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
// Write the chain state to disk, if necessary, to keep RAM usage down
if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED))
return false;
if (disconnectpool) {
@ -2502,8 +2502,8 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp
}
int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3;
LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal);
// Write the chain state to disk, if necessary.
if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
// Write the chain state to disk, if necessary, to keep memory usage down
if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED))
return false;
int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4;
LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal);
@ -2709,7 +2709,7 @@ static void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) {
* we avoid holding cs_main for an extended period of time; the length of this
* call may be quite long during reindexing or a substantial reorg.
*/
bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) {
bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock, bool lastInBatch) {
// Note that while we're often called here from ProcessNewBlock, this is
// far from a guarantee. Things in the P2P/RPC will often end up calling
// us in the middle of ProcessNewBlock - do not assume pblock is set
@ -2767,12 +2767,23 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
pindexMostWork = nullptr;
}
pindexNewTip = chainActive.Tip();
auto wantsAnotherRound = !pindexNewTip || (starting_tip && CBlockIndexWorkComparator()(pindexNewTip, starting_tip));
// flush before we send any signals:
auto flushMode = !lastInBatch || !blocks_connected || IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS;
auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST
&& flushMode == FlushStateMode::ALWAYS && !wantsAnotherRound && pindexNewTip == pindexMostWork;
if (!FlushStateToDisk(chainparams, state, flushMode, 0, diskSync))
return error("Unable to flush after ActivateBestChainStep");
for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) {
assert(trace.pblock && trace.pindex);
GetMainSignals().BlockConnected(trace.pblock, trace.pindex, trace.conflictedTxs);
}
} while (!chainActive.Tip() || (starting_tip && CBlockIndexWorkComparator()(chainActive.Tip(), starting_tip)));
if (!wantsAnotherRound)
break;
} while (true);
if (!blocks_connected) return true;
const CBlockIndex* pindexFork = chainActive.FindFork(starting_tip);
@ -2802,11 +2813,7 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
auto& consensus = chainparams.GetConsensus();
CheckBlockIndex(consensus);
auto flushMode = IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS;
auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST
&& flushMode == FlushStateMode::ALWAYS;
return FlushStateToDisk(chainparams, state, flushMode, 0, diskSync);
return true;
}
bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) {
@ -3042,7 +3049,8 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
}
if (!fKnown) {
while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) {
const static int64_t maxBlockfileSize = std::max<int64_t>(BLOCKFILE_CHUNK_SIZE, gArgs.GetArg("-maxblockfilesize", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U) << 20U);
while (vinfoBlockFile[nFile].nSize + nAddSize >= maxBlockfileSize) {
nFile++;
if (vinfoBlockFile.size() <= nFile) {
vinfoBlockFile.resize(nFile + 1);
@ -3576,7 +3584,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali
return true;
}
bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock)
bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock, bool lastInBatch)
{
AssertLockNotHeld(cs_main);
@ -3603,7 +3611,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons
NotifyHeaderTip();
CValidationState state; // Only used to report errors, not invalidity - ignore it
if (!g_chainstate.ActivateBestChain(state, chainparams, pblock)) {
if (!g_chainstate.ActivateBestChain(state, chainparams, pblock, lastInBatch)) {
return error("%s: ActivateBestChain failed (%s)", __func__, FormatStateMessage(state));
}

8
src/validation.h
View file

@ -74,9 +74,9 @@ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336;
/** Maximum kilobytes for transactions to store for processing during reorg */
static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000;
/** The maximum size of a blk?????.dat file (since 0.8) */
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
static const unsigned int DEFAULT_MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x800000; // 8 MiB
/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
@ -244,7 +244,7 @@ static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024;
* @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call
* @return True if state.IsValid()
*/
bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) LOCKS_EXCLUDED(cs_main);
bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock, bool lastInBatch=true) LOCKS_EXCLUDED(cs_main);
/**
* Process incoming block headers.
@ -473,7 +473,7 @@ public:
bool LoadBlockIndex(const Consensus::Params& consensus_params, CBlockTreeDB& blocktree) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock);
bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock, bool lastInBatch=true);
/**
* If a block header hasn't already been seen, call CheckBlockHeader on it, ensure

4
src/wallet/rpcwallet.cpp
View file

@ -4499,6 +4499,10 @@ UniValue generate(const JSONRPCRequest& request)
);
}
if (IsInitialBlockDownload()) {
throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "generate is not available during the initial block download");
}
int num_generate = request.params[0].get_int();
uint64_t max_tries = 1000000;
if (!request.params[1].isNull()) {

2
src/wallet/test/wallet_tests.cpp
View file

@ -39,7 +39,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup)
// Cap last block file size, and mine new block in a new block file.
CBlockIndex* const nullBlock = nullptr;
CBlockIndex* oldTip = chainActive.Tip();
GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE;
GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = DEFAULT_MAX_BLOCKFILE_SIZE;
CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey()));
CBlockIndex* newTip = chainActive.Tip();