diff --git a/.travis.yml b/.travis.yml index c09303081..267bf9e17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,7 +71,7 @@ jobs: dist: xenial language: minimal git: - depth: 3 + clone: false install: - mkdir -p testrun && cd testrun - curl http://build.lbry.io/lbrycrd/${TRAVIS_BRANCH}/lbrycrd-${NAME}-test.zip -o temp.zip diff --git a/src/init.cpp b/src/init.cpp index 5bed2e4ca..e625a5817 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -370,6 +370,7 @@ void SetupServerArgs() gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS); gArgs.AddArg("-loadblock=", "Imports blocks from external blk000??.dat file on startup", false, OptionsCategory::OPTIONS); + gArgs.AddArg("-maxblockfilesize=", strprintf("Keep the block files below n megabytes each. (default: %d, minimum: 8)", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U), false, OptionsCategory::OPTIONS); gArgs.AddArg("-maxmempool=", strprintf("Keep the transaction memory pool below megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), false, OptionsCategory::OPTIONS); gArgs.AddArg("-maxorphantx=", strprintf("Keep at most unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), false, OptionsCategory::OPTIONS); gArgs.AddArg("-mempoolexpiry=", strprintf("Do not keep transactions in the mempool longer than hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), false, OptionsCategory::OPTIONS); @@ -1427,7 +1428,7 @@ bool AppInitMain() // however, we want the claimtrie cache to be larger than the others int64_t nBlockTreeDBCache = std::min(nTotalCache / 4, nMaxBlockDBCache << 20); - int64_t nCoinDBCache = std::min(nTotalCache / 8, nMaxCoinsDBCache << 20); + int64_t nCoinDBCache = std::min(nTotalCache / 4, nMaxCoinsDBCache << 20); int64_t nClaimtrieCache = nTotalCache / 4; nTotalCache -= nBlockTreeDBCache; nTotalCache -= nCoinDBCache; diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index aee6fbee1..9745cee2c 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -288,7 +288,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, nConf += confAvg[periodTarget - 1][bucket]; totalNum += txCtAvg[bucket]; failNum += failAvg[periodTarget - 1][bucket]; - for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) + const auto maxConfirms = GetMaxConfirms(); + for (unsigned int confct = confTarget; confct < maxConfirms; ++confct) extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; extraNum += oldUnconfTxs[bucket]; // If we have enough transaction data points in this range of buckets, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 7875f05e4..c0778d491 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -142,9 +142,10 @@ UniValue generateBlocks(std::shared_ptr coinbaseScript, int nGen continue; } std::shared_ptr shared_pblock = std::make_shared(*pblock); - if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr)) + auto lastInBatch = ++nHeight >= nHeightEnd; + if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr, lastInBatch)) throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); - ++nHeight; + blockHashes.push_back(pblock->GetHash().GetHex()); //mark script as important because it was used at least for one coinbase output if the script came from the wallet @@ -173,6 +174,10 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) + HelpExampleCli("generatetoaddress", "11 \"myaddress\"") ); + if (IsInitialBlockDownload()) { + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "generatetoaddress is not available during the initial block download"); + } + int nGenerate = request.params[0].get_int(); uint64_t nMaxTries = 1000000; if (!request.params[2].isNull()) { diff --git a/src/test/claimtriefixture.cpp b/src/test/claimtriefixture.cpp index 41ea4e3b6..1af111bc0 100644 --- a/src/test/claimtriefixture.cpp +++ b/src/test/claimtriefixture.cpp @@ -192,7 +192,7 @@ bool ClaimTrieChainFixture::CreateBlock(const std::unique_ptr& p break; } } - auto success = ProcessNewBlock(Params(), std::make_shared(*pblock), true, nullptr); + auto success = ProcessNewBlock(Params(), std::make_shared(*pblock), true, nullptr, false); return success && pblock->GetHash() == chainActive.Tip()->GetBlockHash(); } diff --git a/src/test/claimtriehashfork_tests.cpp b/src/test/claimtriehashfork_tests.cpp index 3be8d78d5..39c1b4938 100644 --- a/src/test/claimtriehashfork_tests.cpp +++ b/src/test/claimtriehashfork_tests.cpp @@ -395,7 +395,7 @@ BOOST_AUTO_TEST_CASE(bogus_claimtrie_hash_test) break; } } - bool success = ProcessNewBlock(Params(), std::make_shared(pblockTemp->block), true, nullptr); + bool success = ProcessNewBlock(Params(), std::make_shared(pblockTemp->block), true, nullptr, false); // will process , but will not be connected BOOST_CHECK(success); BOOST_CHECK(pblockTemp->block.GetHash() != chainActive.Tip()->GetBlockHash()); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 017e9616d..1bb83a144 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -254,7 +254,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) #endif } std::shared_ptr shared_pblock = std::make_shared(*pblock); - BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); + BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr, false)); } LOCK(cs_main); diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 2e1cdd9f7..22b9f0021 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -230,7 +230,7 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector& while (!CheckProofOfWork(block.GetPoWHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr shared_pblock = std::make_shared(block); - ProcessNewBlock(chainparams, shared_pblock, true, nullptr); + ProcessNewBlock(chainparams, shared_pblock, true, nullptr, false); CBlock result = block; return result; diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 8cf0ac44b..00e0a487d 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) BOOST_CHECK(ProcessNewBlockHeaders(headers, state, Params())); // Connect the genesis block and drain any outstanding events - ProcessNewBlock(Params(), std::make_shared(Params().GenesisBlock()), true, &ignored); + ProcessNewBlock(Params(), std::make_shared(Params().GenesisBlock()), true, &ignored, false); SyncWithValidationInterfaceQueue(); // subscribe to events (this subscriber will validate event ordering) @@ -159,13 +159,13 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) bool ignored; for (int i = 0; i < 1000; i++) { auto block = blocks[GetRand(blocks.size() - 1)]; - ProcessNewBlock(Params(), block, true, &ignored); + ProcessNewBlock(Params(), block, true, &ignored, false); } // to make sure that eventually we process the full chain - do it here for (auto block : blocks) { if (block->vtx.size() == 1) { - bool processed = ProcessNewBlock(Params(), block, true, &ignored); + bool processed = ProcessNewBlock(Params(), block, true, &ignored, false); assert(processed); } } diff --git a/src/txdb.cpp b/src/txdb.cpp index 7d4bb57c8..473fee9ce 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -256,7 +256,8 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const CCoinsViewDBCursor::CCoinsViewDBCursor(const uint256 &hashBlockIn, const CCoinsViewDB* view) : CCoinsViewCursor(hashBlockIn), owner(view), - query(owner->db << "SELECT txID, txN, isCoinbase, blockHeight, amount, script FROM unspent") + query(owner->db << "SELECT txID, txN, isCoinbase, blockHeight, amount, script " + "FROM unspent ORDER BY txID ASC, txN ASC") { iter = query.begin(); } diff --git a/src/txdb.h b/src/txdb.h index df2b4098e..11c3cdf2e 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -60,7 +60,7 @@ static const int64_t nMinDbCache = 4; //! Max memory allocated to block tree DB specific cache static const int64_t nMaxBlockDBCache = 260; //! Max memory allocated to coin DB specific cache (MiB) -static const int64_t nMaxCoinsDBCache = 40; +static const int64_t nMaxCoinsDBCache = 200; struct CDiskTxPos : public CDiskBlockPos diff --git a/src/validation.cpp b/src/validation.cpp index d4de779d0..e04673bb6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2355,8 +2355,8 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha assert(pindexDelete->pprev->hashClaimTrie == trieCache.getMerkleHash()); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); - // Write the chain state to disk, if necessary. - if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) + // Write the chain state to disk, if necessary, to keep RAM usage down + if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) return false; if (disconnectpool) { @@ -2502,8 +2502,8 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal); - // Write the chain state to disk, if necessary. - if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) + // Write the chain state to disk, if necessary, to keep memory usage down + if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) return false; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); @@ -2709,7 +2709,7 @@ static void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { * we avoid holding cs_main for an extended period of time; the length of this * call may be quite long during reindexing or a substantial reorg. */ -bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr pblock) { +bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr pblock, bool lastInBatch) { // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set @@ -2767,12 +2767,23 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& pindexMostWork = nullptr; } pindexNewTip = chainActive.Tip(); + auto wantsAnotherRound = !pindexNewTip || (starting_tip && CBlockIndexWorkComparator()(pindexNewTip, starting_tip)); + + // flush before we send any signals: + auto flushMode = !lastInBatch || !blocks_connected || IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS; + auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST + && flushMode == FlushStateMode::ALWAYS && !wantsAnotherRound && pindexNewTip == pindexMostWork; + if (!FlushStateToDisk(chainparams, state, flushMode, 0, diskSync)) + return error("Unable to flush after ActivateBestChainStep"); for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { assert(trace.pblock && trace.pindex); GetMainSignals().BlockConnected(trace.pblock, trace.pindex, trace.conflictedTxs); } - } while (!chainActive.Tip() || (starting_tip && CBlockIndexWorkComparator()(chainActive.Tip(), starting_tip))); + + if (!wantsAnotherRound) + break; + } while (true); if (!blocks_connected) return true; const CBlockIndex* pindexFork = chainActive.FindFork(starting_tip); @@ -2802,11 +2813,7 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& auto& consensus = chainparams.GetConsensus(); CheckBlockIndex(consensus); - - auto flushMode = IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS; - auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST - && flushMode == FlushStateMode::ALWAYS; - return FlushStateToDisk(chainparams, state, flushMode, 0, diskSync); + return true; } bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr pblock) { @@ -3042,7 +3049,8 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int } if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + const static int64_t maxBlockfileSize = std::max(BLOCKFILE_CHUNK_SIZE, gArgs.GetArg("-maxblockfilesize", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U) << 20U); + while (vinfoBlockFile[nFile].nSize + nAddSize >= maxBlockfileSize) { nFile++; if (vinfoBlockFile.size() <= nFile) { vinfoBlockFile.resize(nFile + 1); @@ -3576,7 +3584,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, CVali return true; } -bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr pblock, bool fForceProcessing, bool *fNewBlock) +bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr pblock, bool fForceProcessing, bool *fNewBlock, bool lastInBatch) { AssertLockNotHeld(cs_main); @@ -3603,7 +3611,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr pblock, bool fForceProcessing, bool* fNewBlock) LOCKS_EXCLUDED(cs_main); +bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr pblock, bool fForceProcessing, bool* fNewBlock, bool lastInBatch=true) LOCKS_EXCLUDED(cs_main); /** * Process incoming block headers. @@ -473,7 +473,7 @@ public: bool LoadBlockIndex(const Consensus::Params& consensus_params, CBlockTreeDB& blocktree) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr pblock); + bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr pblock, bool lastInBatch=true); /** * If a block header hasn't already been seen, call CheckBlockHeader on it, ensure diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index ffede598a..8ab89857a 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -4499,6 +4499,10 @@ UniValue generate(const JSONRPCRequest& request) ); } + if (IsInitialBlockDownload()) { + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "generate is not available during the initial block download"); + } + int num_generate = request.params[0].get_int(); uint64_t max_tries = 1000000; if (!request.params[1].isNull()) { diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 96c537220..3db702aaf 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -39,7 +39,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) // Cap last block file size, and mine new block in a new block file. CBlockIndex* const nullBlock = nullptr; CBlockIndex* oldTip = chainActive.Tip(); - GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE; + GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = DEFAULT_MAX_BLOCKFILE_SIZE; CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = chainActive.Tip();