LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues29 Pull requests1 Projects Releases32 Packages Wiki Activity

flush more, disk sync less

Browse source
This commit is contained in:
Brannon King 2020-02-12 11:40:14 -07:00
parent 873c597ba6
commit 4c342fae29
4 changed files with 25 additions and 31 deletions

4
src/init.cpp
View file

@ -241,7 +241,7 @@ void Shutdown()
// FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing
if (pcoinsTip != nullptr) { if (pcoinsTip != nullptr) {
FlushStateToDisk(); FlushStateToDisk(true);
} }
// After there are no more peers/RPC left to give us new data which may generate // After there are no more peers/RPC left to give us new data which may generate
@ -257,7 +257,7 @@ void Shutdown()
{ {
LOCK(cs_main); LOCK(cs_main);
if (pcoinsTip != nullptr) { if (pcoinsTip != nullptr) {
FlushStateToDisk(); FlushStateToDisk(true);
} }
pcoinsTip.reset(); pcoinsTip.reset();
pcoinscatcher.reset(); pcoinscatcher.reset();

17
src/txdb.cpp
View file

@ -115,7 +115,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, boo
auto dbd = db << "DELETE FROM unspent WHERE txID = ? AND txN = ?"; auto dbd = db << "DELETE FROM unspent WHERE txID = ? AND txN = ?";
auto dbi = db << "INSERT OR REPLACE INTO unspent VALUES(?,?,?,?,?,?,?)"; auto dbi = db << "INSERT OR REPLACE INTO unspent VALUES(?,?,?,?,?,?,?)";
for (auto it = mapCoins.begin(); it != mapCoins.end();) { for (auto it = mapCoins.begin(); it != mapCoins.end(); it = mapCoins.erase(it)) {
if (it->second.flags & CCoinsCacheEntry::DIRTY) { if (it->second.flags & CCoinsCacheEntry::DIRTY) {
if (it->second.coin.IsSpent()) { if (it->second.coin.IsSpent()) {
// at present the "IsSpent" flag is used for both "spent" and "block going backwards" // at present the "IsSpent" flag is used for both "spent" and "block going backwards"
@ -135,10 +135,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, boo
} }
changed++; changed++;
} }
count++; if (crash_simulate && ++count % 200000 == 0) {
auto itOld = it++;
mapCoins.erase(itOld);
if (crash_simulate && count % 200000 == 0) {
static FastRandomContext rng; static FastRandomContext rng;
if (rng.randrange(crash_simulate) == 0) { if (rng.randrange(crash_simulate) == 0) {
LogPrintf("Simulating a crash. Goodbye.\n"); LogPrintf("Simulating a crash. Goodbye.\n");
@ -153,14 +150,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, boo
auto code = sqlite::commit(db); auto code = sqlite::commit(db);
if (code != SQLITE_OK) { if (code != SQLITE_OK) {
LogPrint(BCLog::COINDB, "Error committing transaction outputs changes to coin database. SQLite error: %d\n", code); LogPrintf("%s: Error committing coins info to database. SQLite error: %d\n", __func__, code);
return false; return false;
} }
LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); LogPrint(BCLog::COINDB, "Committed %zu changed transaction outputs (out of %zu) to coin database...\n", changed, count);
if (sync) { if (sync) {
auto code = sqlite::sync(db); code = sqlite::sync(db);
if (code != SQLITE_OK) { if (code != SQLITE_OK) {
LogPrint(BCLog::COINDB, "Error syncing coin database. SQLite error: %d\n", code); LogPrintf("%s: Error syncing coin database. SQLite error: %d\n", __func__, code);
return false; return false;
} }
} }
@ -340,7 +337,7 @@ bool CBlockTreeDB::BatchWrite(const std::vector<std::pair<int, const CBlockFileI
} }
// by Sync they mean disk sync: // by Sync they mean disk sync:
if (sync) { if (sync) {
auto code = sqlite::sync(db); code = sqlite::sync(db);
if (code != SQLITE_OK) { if (code != SQLITE_OK) {
LogPrintf("%s: Error syncing block database. SQLite error: %d\n", __func__, code); LogPrintf("%s: Error syncing block database. SQLite error: %d\n", __func__, code);
return false; return false;

33
src/validation.cpp
View file

@ -156,7 +156,7 @@ enum class FlushStateMode {
}; };
// See definition for documentation // See definition for documentation
static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0); static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0, bool syncToDisk=false);
static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight);
static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight);
bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr);
@ -2117,7 +2117,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
* If FlushStateMode::NONE is used, then FlushStateToDisk(...) won't do anything * If FlushStateMode::NONE is used, then FlushStateToDisk(...) won't do anything
* besides checking if we need to prune. * besides checking if we need to prune.
*/ */
bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &state, FlushStateMode mode, int nManualPruneHeight) { bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &state,
FlushStateMode mode, int nManualPruneHeight, bool syncToDisk) {
int64_t nMempoolUsage = mempool.DynamicMemoryUsage(); int64_t nMempoolUsage = mempool.DynamicMemoryUsage();
LOCK(cs_main); LOCK(cs_main);
static int64_t nLastWrite = 0; static int64_t nLastWrite = 0;
@ -2186,7 +2187,7 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
vBlocks.push_back(*it); vBlocks.push_back(*it);
setDirtyBlockIndex.erase(it++); setDirtyBlockIndex.erase(it++);
} }
if (!pblocktree->BatchWrite(vFiles, nLastBlockFile, vBlocks, mode == FlushStateMode::ALWAYS)) { if (!pblocktree->BatchWrite(vFiles, nLastBlockFile, vBlocks, syncToDisk)) {
return AbortNode(state, "Failed to write to block index database"); return AbortNode(state, "Failed to write to block index database");
} }
} }
@ -2204,10 +2205,10 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// overwrite one. Still, use a conservative safety factor of 2. // overwrite one. Still, use a conservative safety factor of 2.
if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize()))
return state.Error("out of disk space"); return state.Error("out of disk space");
if (mode == FlushStateMode::ALWAYS && !pclaimTrie->SyncToDisk()) if (syncToDisk && !pclaimTrie->SyncToDisk())
return state.Error("Failed to write to claim trie database"); return state.Error("Failed to write to claim trie database");
// Flush the chainstate (which may refer to block index entries). // Flush the chainstate (which may refer to block index entries).
if (!pcoinsTip->Flush(mode == FlushStateMode::ALWAYS)) if (!pcoinsTip->Flush(syncToDisk))
return AbortNode(state, "Failed to write to coin database"); return AbortNode(state, "Failed to write to coin database");
nLastFlush = nNow; nLastFlush = nNow;
full_flush_completed = true; full_flush_completed = true;
@ -2224,10 +2225,10 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
return true; return true;
} }
void FlushStateToDisk() { void FlushStateToDisk(bool diskSync) {
CValidationState state; CValidationState state;
const CChainParams& chainparams = Params(); const CChainParams& chainparams = Params();
if (!FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) { if (!FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS, 0, diskSync)) {
LogPrintf("%s: failed to flush state (%s)\n", __func__, FormatStateMessage(state)); LogPrintf("%s: failed to flush state (%s)\n", __func__, FormatStateMessage(state));
} }
} }
@ -2355,7 +2356,7 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha
} }
LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI);
// Write the chain state to disk, if necessary. // Write the chain state to disk, if necessary.
if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
return false; return false;
if (disconnectpool) { if (disconnectpool) {
@ -2502,7 +2503,7 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp
int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3;
LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal); LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal);
// Write the chain state to disk, if necessary. // Write the chain state to disk, if necessary.
if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
return false; return false;
int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4;
LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal);
@ -2775,11 +2776,11 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
if (!blocks_connected) return true; if (!blocks_connected) return true;
const CBlockIndex* pindexFork = chainActive.FindFork(starting_tip); const CBlockIndex* pindexFork = chainActive.FindFork(starting_tip);
bool fInitialDownload = IsInitialBlockDownload();
// Notify external listeners about the new tip. // Notify external listeners about the new tip.
// Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected // Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected
if (pindexFork != pindexNewTip) { if (pindexFork != pindexNewTip) {
bool fInitialDownload = IsInitialBlockDownload();
// Notify ValidationInterface subscribers // Notify ValidationInterface subscribers
GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload);
@ -2802,14 +2803,10 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
auto& consensus = chainparams.GetConsensus(); auto& consensus = chainparams.GetConsensus();
CheckBlockIndex(consensus); CheckBlockIndex(consensus);
auto flushMode = FlushStateMode::PERIODIC; auto flushMode = IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS;
if (pindexNewTip && chainparams.NetworkIDString() != CBaseChainParams::REGTEST auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST
&& pindexNewTip->nTime + consensus.nPowTargetSpacing > GetAdjustedTime()) { && flushMode == FlushStateMode::ALWAYS;
// trying to ensure that we flush to disk after new blocks when we're caught up to the chain return FlushStateToDisk(chainparams, state, flushMode, 0, diskSync);
// they're technically allowed to be two hours late, but experience says one minute is more likely
flushMode = FlushStateMode::ALWAYS;
}
return FlushStateToDisk(chainparams, state, flushMode);
} }
bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) {

2
src/validation.h
View file

@ -291,7 +291,7 @@ void PruneOneBlockFile(const int fileNumber);
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune);
/** Flush all state, indexes and buffers to disk. */ /** Flush all state, indexes and buffers to disk. */
void FlushStateToDisk(); void FlushStateToDisk(bool diskSync = false);
/** Prune block files and flush state to disk. */ /** Prune block files and flush state to disk. */
void PruneAndFlush(); void PruneAndFlush();
/** Prune block files up to a given height */ /** Prune block files up to a given height */