Merge branch 'add_maxblockfilesize_param'

.travis.yml

@@ -71,7 +71,7 @@ jobs:
       dist: xenial
       language: minimal
       git:
-        depth: 3
+        clone: false
       install:
         - mkdir -p testrun && cd testrun
         - curl http://build.lbry.io/lbrycrd/${TRAVIS_BRANCH}/lbrycrd-${NAME}-test.zip -o temp.zip

src/init.cpp

@@ -370,6 +370,7 @@ void SetupServerArgs()
     gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS);
     gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", false, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-maxblockfilesize=<n>", strprintf("Keep the block files below n megabytes each. (default: %d, minimum: 8)", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), false, OptionsCategory::OPTIONS);
@@ -1427,7 +1428,7 @@ bool AppInitMain()
     // however, we want the claimtrie cache to be larger than the others
 
     int64_t nBlockTreeDBCache = std::min(nTotalCache / 4, nMaxBlockDBCache << 20);
-    int64_t nCoinDBCache = std::min(nTotalCache / 8, nMaxCoinsDBCache << 20);
+    int64_t nCoinDBCache = std::min(nTotalCache / 4, nMaxCoinsDBCache << 20);
     int64_t nClaimtrieCache = nTotalCache / 4;
     nTotalCache -= nBlockTreeDBCache;
     nTotalCache -= nCoinDBCache;

src/policy/fees.cpp

@@ -288,7 +288,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal,
         nConf += confAvg[periodTarget - 1][bucket];
         totalNum += txCtAvg[bucket];
         failNum += failAvg[periodTarget - 1][bucket];
-        for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++)
+        const auto maxConfirms = GetMaxConfirms();
+        for (unsigned int confct = confTarget; confct < maxConfirms; ++confct)
             extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket];
         extraNum += oldUnconfTxs[bucket];
         // If we have enough transaction data points in this range of buckets,

src/rpc/mining.cpp

@@ -142,9 +142,10 @@ UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGen
             continue;
         }
         std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock);
-        if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr))
+        auto lastInBatch = ++nHeight >= nHeightEnd;
+        if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr, lastInBatch))
             throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted");
-        ++nHeight;
+
         blockHashes.push_back(pblock->GetHash().GetHex());
 
         //mark script as important because it was used at least for one coinbase output if the script came from the wallet
@@ -173,6 +174,10 @@ static UniValue generatetoaddress(const JSONRPCRequest& request)
             + HelpExampleCli("generatetoaddress", "11 \"myaddress\"")
         );
 
+    if (IsInitialBlockDownload()) {
+        throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "generatetoaddress is not available during the initial block download");
+    }
+
     int nGenerate = request.params[0].get_int();
     uint64_t nMaxTries = 1000000;
     if (!request.params[2].isNull()) {

src/test/claimtriefixture.cpp

@@ -192,7 +192,7 @@ bool ClaimTrieChainFixture::CreateBlock(const std::unique_ptr<CBlockTemplate>& p
                 break;
         }
     }
-    auto success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(*pblock), true, nullptr);
+    auto success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(*pblock), true, nullptr, false);
     return success && pblock->GetHash() == chainActive.Tip()->GetBlockHash();
 }
 

src/test/claimtriehashfork_tests.cpp

@@ -395,7 +395,7 @@ BOOST_AUTO_TEST_CASE(bogus_claimtrie_hash_test)
             break;
         }
     }
-    bool success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(pblockTemp->block), true, nullptr);
+    bool success = ProcessNewBlock(Params(), std::make_shared<const CBlock>(pblockTemp->block), true, nullptr, false);
     // will process , but will not be connected
     BOOST_CHECK(success);
     BOOST_CHECK(pblockTemp->block.GetHash() != chainActive.Tip()->GetBlockHash());

src/test/miner_tests.cpp

@@ -254,7 +254,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
 #endif
         }
         std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock);
-        BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr));
+        BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr, false));
     }
 
     LOCK(cs_main);

src/test/test_bitcoin.cpp

@@ -230,7 +230,7 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>&
     while (!CheckProofOfWork(block.GetPoWHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce;
 
     std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block);
-    ProcessNewBlock(chainparams, shared_pblock, true, nullptr);
+    ProcessNewBlock(chainparams, shared_pblock, true, nullptr, false);
 
     CBlock result = block;
     return result;

src/test/validation_block_tests.cpp

@@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering)
     BOOST_CHECK(ProcessNewBlockHeaders(headers, state, Params()));
 
     // Connect the genesis block and drain any outstanding events
-    ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored);
+    ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored, false);
     SyncWithValidationInterfaceQueue();
 
     // subscribe to events (this subscriber will validate event ordering)
@@ -159,13 +159,13 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering)
             bool ignored;
             for (int i = 0; i < 1000; i++) {
                 auto block = blocks[GetRand(blocks.size() - 1)];
-                ProcessNewBlock(Params(), block, true, &ignored);
+                ProcessNewBlock(Params(), block, true, &ignored, false);
             }
 
             // to make sure that eventually we process the full chain - do it here
             for (auto block : blocks) {
                 if (block->vtx.size() == 1) {
-                    bool processed = ProcessNewBlock(Params(), block, true, &ignored);
+                    bool processed = ProcessNewBlock(Params(), block, true, &ignored, false);
                     assert(processed);
                 }
             }

src/txdb.cpp

@@ -256,7 +256,8 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const
 
 CCoinsViewDBCursor::CCoinsViewDBCursor(const uint256 &hashBlockIn, const CCoinsViewDB* view)
         : CCoinsViewCursor(hashBlockIn), owner(view),
-        query(owner->db << "SELECT txID, txN, isCoinbase, blockHeight, amount, script FROM unspent")
+        query(owner->db << "SELECT txID, txN, isCoinbase, blockHeight, amount, script "
+                           "FROM unspent ORDER BY txID ASC, txN ASC")
 {
     iter = query.begin();
 }

src/txdb.h

@@ -60,7 +60,7 @@ static const int64_t nMinDbCache = 4;
 //! Max memory allocated to block tree DB specific cache
 static const int64_t nMaxBlockDBCache = 260;
 //! Max memory allocated to coin DB specific cache (MiB)
-static const int64_t nMaxCoinsDBCache = 40;
+static const int64_t nMaxCoinsDBCache = 200;
 
 
 struct CDiskTxPos : public CDiskBlockPos

src/validation.cpp

@@ -2355,8 +2355,8 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha
         assert(pindexDelete->pprev->hashClaimTrie == trieCache.getMerkleHash());
     }
     LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI);
-    // Write the chain state to disk, if necessary.
+    // Write the chain state to disk, if necessary, to keep RAM usage down
-    if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
+    if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED))
         return false;
 
     if (disconnectpool) {
@@ -2502,8 +2502,8 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp
     }
     int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3;
     LogPrint(BCLog::BENCH, "  - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal);
-    // Write the chain state to disk, if necessary.
+    // Write the chain state to disk, if necessary, to keep memory usage down
-    if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
+    if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED))
         return false;
     int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4;
     LogPrint(BCLog::BENCH, "  - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal);
@@ -2709,7 +2709,7 @@ static void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) {
  * we avoid holding cs_main for an extended period of time; the length of this
  * call may be quite long during reindexing or a substantial reorg.
  */
-bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) {
+bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock, bool lastInBatch) {
     // Note that while we're often called here from ProcessNewBlock, this is
     // far from a guarantee. Things in the P2P/RPC will often end up calling
     // us in the middle of ProcessNewBlock - do not assume pblock is set
@@ -2767,12 +2767,23 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
                     pindexMostWork = nullptr;
                 }
                 pindexNewTip = chainActive.Tip();
+                auto wantsAnotherRound = !pindexNewTip || (starting_tip && CBlockIndexWorkComparator()(pindexNewTip, starting_tip));
+
+                // flush before we send any signals:
+                auto flushMode = !lastInBatch || !blocks_connected || IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS;
+                auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST
+                                && flushMode == FlushStateMode::ALWAYS && !wantsAnotherRound && pindexNewTip == pindexMostWork;
+                if (!FlushStateToDisk(chainparams, state, flushMode, 0, diskSync))
+                    return error("Unable to flush after ActivateBestChainStep");
 
                 for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) {
                     assert(trace.pblock && trace.pindex);
                     GetMainSignals().BlockConnected(trace.pblock, trace.pindex, trace.conflictedTxs);
                 }
-            } while (!chainActive.Tip() || (starting_tip && CBlockIndexWorkComparator()(chainActive.Tip(), starting_tip)));
+
+                if (!wantsAnotherRound)
+                    break;
+            } while (true);
             if (!blocks_connected) return true;
 
             const CBlockIndex* pindexFork = chainActive.FindFork(starting_tip);
@@ -2802,11 +2813,7 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
 
     auto& consensus = chainparams.GetConsensus();
     CheckBlockIndex(consensus);
-
+    return true;
-    auto flushMode = IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS;
-    auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST
-                    && flushMode == FlushStateMode::ALWAYS;
-    return FlushStateToDisk(chainparams, state, flushMode, 0, diskSync);
 }
 
 bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) {
@@ -3042,7 +3049,8 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
     }
 
     if (!fKnown) {
-        while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) {
+        const static int64_t maxBlockfileSize = std::max<int64_t>(BLOCKFILE_CHUNK_SIZE, gArgs.GetArg("-maxblockfilesize", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U) << 20U);
+        while (vinfoBlockFile[nFile].nSize + nAddSize >= maxBlockfileSize) {
             nFile++;
             if (vinfoBlockFile.size() <= nFile) {
                 vinfoBlockFile.resize(nFile + 1);
@@ -3576,7 +3584,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali
     return true;
 }
 
-bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock)
+bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock, bool lastInBatch)
 {
     AssertLockNotHeld(cs_main);
 
@@ -3603,7 +3611,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons
     NotifyHeaderTip();
 
     CValidationState state; // Only used to report errors, not invalidity - ignore it
-    if (!g_chainstate.ActivateBestChain(state, chainparams, pblock)) {
+    if (!g_chainstate.ActivateBestChain(state, chainparams, pblock, lastInBatch)) {
         return error("%s: ActivateBestChain failed (%s)", __func__, FormatStateMessage(state));
     }
 

src/validation.h

@@ -74,9 +74,9 @@ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336;
 /** Maximum kilobytes for transactions to store for processing during reorg */
 static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000;
 /** The maximum size of a blk?????.dat file (since 0.8) */
-static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
+static const unsigned int DEFAULT_MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
 /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
-static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
+static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x800000; // 8 MiB
 /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
 static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
 
@@ -244,7 +244,7 @@ static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024;
  * @param[out]  fNewBlock A boolean which is set to indicate if the block was first received via this call
  * @return True if state.IsValid()
  */
-bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) LOCKS_EXCLUDED(cs_main);
+bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock, bool lastInBatch=true) LOCKS_EXCLUDED(cs_main);
 
 /**
  * Process incoming block headers.
@@ -473,7 +473,7 @@ public:
 
     bool LoadBlockIndex(const Consensus::Params& consensus_params, CBlockTreeDB& blocktree) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
 
-    bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock);
+    bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock, bool lastInBatch=true);
 
     /**
      * If a block header hasn't already been seen, call CheckBlockHeader on it, ensure

src/wallet/rpcwallet.cpp

@@ -4499,6 +4499,10 @@ UniValue generate(const JSONRPCRequest& request)
         );
     }
 
+    if (IsInitialBlockDownload()) {
+        throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "generate is not available during the initial block download");
+    }
+
     int num_generate = request.params[0].get_int();
     uint64_t max_tries = 1000000;
     if (!request.params[1].isNull()) {

src/wallet/test/wallet_tests.cpp

@@ -39,7 +39,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup)
     // Cap last block file size, and mine new block in a new block file.
     CBlockIndex* const nullBlock = nullptr;
     CBlockIndex* oldTip = chainActive.Tip();
-    GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE;
+    GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = DEFAULT_MAX_BLOCKFILE_SIZE;
     CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey()));
     CBlockIndex* newTip = chainActive.Tip();