From 386a6b62a8a1db9dd0f354cb95b7585f555c7e5d Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Fri, 9 Mar 2018 12:12:43 +0800 Subject: [PATCH 1/3] Allow to optional specify the directory for the blocks storage --- src/init.cpp | 9 +++++++-- src/qt/bitcoin.cpp | 2 +- src/txdb.cpp | 2 +- src/util.cpp | 33 +++++++++++++++++++++++++++++++++ src/util.h | 1 + src/validation.cpp | 12 ++++++------ src/validation.h | 2 +- 7 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index f763c3a43..eb8beda8d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -333,6 +333,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-version", _("Print version and exit")); strUsage += HelpMessageOpt("-alertnotify=", _("Execute command when a relevant alert is received or we see a really long fork (%s in cmd is replaced by message)")); strUsage +=HelpMessageOpt("-assumevalid=", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex())); + strUsage += HelpMessageOpt("-blocksdir=", _("Specify blocks directory (default: /blocks)")); strUsage += HelpMessageOpt("-blocknotify=", _("Execute command when the best block changes (%s in cmd is replaced by block hash)")); strUsage += HelpMessageOpt("-blockreconstructionextratxn=", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); if (showDebug) @@ -596,7 +597,7 @@ void CleanupBlockRevFiles() // Remove the rev files immediately and insert the blk file paths into an // ordered map keyed by block file index. LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - fs::path blocksdir = GetDataDir() / "blocks"; + fs::path blocksdir = GetBlocksDir(); for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { if (fs::is_regular_file(*it) && it->path().filename().string().length() == 12 && @@ -908,6 +909,10 @@ bool AppInitParameterInteraction() // also see: InitParameterInteraction() + if (!fs::is_directory(GetBlocksDir(false))) { + return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.\n"), gArgs.GetArg("-blocksdir", "").c_str())); + } + // if using block pruning, then disallow txindex if (gArgs.GetArg("-prune", 0)) { if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) @@ -1630,7 +1635,7 @@ bool AppInitMain() // ********************************************************* Step 10: import blocks - if (!CheckDiskSpace()) + if (!CheckDiskSpace() && !CheckDiskSpace(0, true)) return false; // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly. diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index ab381bfb5..a47b4af44 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -617,7 +617,7 @@ int main(int argc, char *argv[]) if (!Intro::pickDataDirectory()) return EXIT_SUCCESS; - /// 6. Determine availability of data directory and parse bitcoin.conf + /// 6. Determine availability of data and blocks directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes if (!fs::is_directory(GetDataDir(false))) { diff --git a/src/txdb.cpp b/src/txdb.cpp index 293d43c7b..f6d60411b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -147,7 +147,7 @@ size_t CCoinsViewDB::EstimateSize() const return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetBlocksDir() / "index", nCacheSize, fMemory, fWipe) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { diff --git a/src/util.cpp b/src/util.cpp index dcf7ed38b..18a020ccd 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -598,10 +598,41 @@ fs::path GetDefaultDataDir() #endif } +static fs::path g_blocks_path_cached; +static fs::path g_blocks_path_cache_net_specific; static fs::path pathCached; static fs::path pathCachedNetSpecific; static CCriticalSection csPathCached; +const fs::path &GetBlocksDir(bool fNetSpecific) +{ + + LOCK(csPathCached); + + fs::path &path = fNetSpecific ? g_blocks_path_cache_net_specific : g_blocks_path_cached; + + // This can be called during exceptions by LogPrintf(), so we cache the + // value so we don't have to do memory allocations after that. + if (!path.empty()) + return path; + + if (gArgs.IsArgSet("-blocksdir")) { + path = fs::system_complete(gArgs.GetArg("-blocksdir", "")); + if (!fs::is_directory(path)) { + path = ""; + return path; + } + } else { + path = GetDataDir(false); + } + if (fNetSpecific) + path /= BaseParams().DataDir(); + + path /= "blocks"; + fs::create_directories(path); + return path; +} + const fs::path &GetDataDir(bool fNetSpecific) { @@ -640,6 +671,8 @@ void ClearDatadirCache() pathCached = fs::path(); pathCachedNetSpecific = fs::path(); + g_blocks_path_cached = fs::path(); + g_blocks_path_cache_net_specific = fs::path(); } fs::path GetConfigFile(const std::string& confPath) diff --git a/src/util.h b/src/util.h index 9490a5678..0c31ac5e6 100644 --- a/src/util.h +++ b/src/util.h @@ -182,6 +182,7 @@ void ReleaseDirectoryLocks(); bool TryCreateDirectories(const fs::path& p); fs::path GetDefaultDataDir(); +const fs::path &GetBlocksDir(bool fNetSpecific = true); const fs::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); fs::path GetConfigFile(const std::string& confPath); diff --git a/src/validation.cpp b/src/validation.cpp index a77362f5d..16bba4f7b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2056,7 +2056,7 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index - if (!CheckDiskSpace(0)) + if (!CheckDiskSpace(0, true)) return state.Error("out of disk space"); // First make sure all block and undo data is flushed to disk. FlushBlockFile(); @@ -2895,7 +2895,7 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int if (nNewChunks > nOldChunks) { if (fPruneMode) fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { + if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) { FILE *file = OpenBlockFile(pos); if (file) { LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); @@ -2928,7 +2928,7 @@ static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, if (nNewChunks > nOldChunks) { if (fPruneMode) fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { + if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) { FILE *file = OpenUndoFile(pos); if (file) { LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); @@ -3604,9 +3604,9 @@ static void FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfte nLastBlockWeCanPrune, count); } -bool CheckDiskSpace(uint64_t nAdditionalBytes) +bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir) { - uint64_t nFreeBytesAvailable = fs::space(GetDataDir()).available; + uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available; // Check for nMinDiskSpace bytes (currently 50MB) if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) @@ -3649,7 +3649,7 @@ static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) { - return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); + return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile); } CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) diff --git a/src/validation.h b/src/validation.h index 99cbfdf1e..ba7a018ec 100644 --- a/src/validation.h +++ b/src/validation.h @@ -254,7 +254,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr, CBlockHeader *first_invalid=nullptr); /** Check whether enough disk space is available for an incoming block */ -bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); +bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false); /** Open a block file (blk?????.dat) */ FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ From f38e4fdb06406f9a1a2562836fab523eb75b5090 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Fri, 9 Mar 2018 12:43:55 +0800 Subject: [PATCH 2/3] QA: Add -blocksdir test --- test/functional/feature_blocksdir.py | 34 ++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 2 files changed, 35 insertions(+) create mode 100755 test/functional/feature_blocksdir.py diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py new file mode 100755 index 000000000..e9d7b9de5 --- /dev/null +++ b/test/functional/feature_blocksdir.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the blocksdir option. +""" + +from test_framework.test_framework import BitcoinTestFramework, initialize_datadir + +import shutil +import os + +class BlocksdirTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + self.stop_node(0) + node0path = os.path.join(self.options.tmpdir, "node0") + shutil.rmtree(node0path) + initialize_datadir(self.options.tmpdir, 0) + self.log.info("Starting with non exiting blocksdir ...") + self.assert_start_raises_init_error(0, ["-blocksdir="+self.options.tmpdir+ "/blocksdir"], "Specified blocks director") + os.mkdir(self.options.tmpdir+ "/blocksdir") + self.log.info("Starting with exiting blocksdir ...") + self.start_node(0, ["-blocksdir="+self.options.tmpdir+ "/blocksdir"]) + self.log.info("mining blocks..") + self.nodes[0].generate(10) + assert(os.path.isfile(self.options.tmpdir+ "/blocksdir/regtest/blocks/blk00000.dat")) + assert(os.path.isdir(self.options.tmpdir+ "/blocksdir/regtest/blocks/index")) + +if __name__ == '__main__': + BlocksdirTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 082191098..14211cd40 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -133,6 +133,7 @@ BASE_SCRIPTS= [ 'p2p_unrequested_blocks.py', 'feature_logging.py', 'p2p_node_network_limited.py', + 'feature_blocksdir.py', 'feature_config_args.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time From a1926362ecb3c354ae338ef7d7020daf78f980c9 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Sun, 11 Mar 2018 12:42:02 +0800 Subject: [PATCH 3/3] -blocksdir: keep blockindex leveldb database in datadir --- src/txdb.cpp | 2 +- test/functional/feature_blocksdir.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index f6d60411b..4a2c32188 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -147,7 +147,7 @@ size_t CCoinsViewDB::EstimateSize() const return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetBlocksDir() / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(gArgs.IsArgSet("-blocksdir") ? GetDataDir() / "blocks" / "index" : GetBlocksDir() / "index", nCacheSize, fMemory, fWipe) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index e9d7b9de5..6ee6cb9a3 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -27,8 +27,8 @@ class BlocksdirTest(BitcoinTestFramework): self.start_node(0, ["-blocksdir="+self.options.tmpdir+ "/blocksdir"]) self.log.info("mining blocks..") self.nodes[0].generate(10) - assert(os.path.isfile(self.options.tmpdir+ "/blocksdir/regtest/blocks/blk00000.dat")) - assert(os.path.isdir(self.options.tmpdir+ "/blocksdir/regtest/blocks/index")) + assert(os.path.isfile(os.path.join(self.options.tmpdir, "blocksdir", "regtest", "blocks", "blk00000.dat"))) + assert(os.path.isdir(os.path.join(self.options.tmpdir, "node0", "regtest", "blocks", "index"))) if __name__ == '__main__': BlocksdirTest().main()