Merge pull request #6221

c257a8c Prune: Support noncontiguous block files (Adam Weiss)

src/init.cpp

@@ -474,25 +474,44 @@ struct CImportingNow
 
 // If we're using -prune with -reindex, then delete block files that will be ignored by the
 // reindex.  Since reindexing works by starting at block file 0 and looping until a blockfile
-// is missing, and since pruning works by deleting the oldest block file first, just check
+// is missing, do the same here to delete any later block files after a gap.  Also delete all
-// for block file 0, and if it doesn't exist, delete all the block files in the
+// rev files since they'll be rewritten by the reindex anyway.  This ensures that vinfoBlockFile
-// directory (since they won't be read by the reindex but will take up disk space).
+// is in sync with what's actually on disk by the time we start downloading, so that pruning
-void DeleteAllBlockFiles()
+// works correctly.
+void CleanupBlockRevFiles()
 {
-    if (boost::filesystem::exists(GetBlockPosFilename(CDiskBlockPos(0, 0), "blk")))
+    using namespace boost::filesystem;
-        return;
+    map<string, path> mapBlockFiles;
 
-    LogPrintf("Removing all blk?????.dat and rev?????.dat files for -reindex with -prune\n");
+    // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
-    boost::filesystem::path blocksdir = GetDataDir() / "blocks";
+    // Remove the rev files immediately and insert the blk file paths into an
-    for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) {
+    // ordered map keyed by block file index.
-        if (is_regular_file(*it)) {
+    LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
-            if ((it->path().filename().string().length() == 12) &&
+    path blocksdir = GetDataDir() / "blocks";
-                (it->path().filename().string().substr(8,4) == ".dat") &&
+    for (directory_iterator it(blocksdir); it != directory_iterator(); it++) {
-                ((it->path().filename().string().substr(0,3) == "blk") ||
+        if (is_regular_file(*it) &&
-                 (it->path().filename().string().substr(0,3) == "rev")))
+            it->path().filename().string().length() == 12 &&
-                boost::filesystem::remove(it->path());
+            it->path().filename().string().substr(8,4) == ".dat")
+        {
+            if (it->path().filename().string().substr(0,3) == "blk")
+                mapBlockFiles[it->path().filename().string().substr(3,5)] = it->path();
+            else if (it->path().filename().string().substr(0,3) == "rev")
+                remove(it->path());
         }
     }
+
+    // Remove all block files that aren't part of a contiguous set starting at
+    // zero by walking the ordered map (keys are block file indices) by
+    // keeping a separate counter.  Once we hit a gap (or if 0 doesn't exist)
+    // start removing block files.
+    int nContigCounter = 0;
+    BOOST_FOREACH(const PAIRTYPE(string, path)& item, mapBlockFiles) {
+        if (atoi(item.first) == nContigCounter) {
+            nContigCounter++;
+            continue;
+        }
+        remove(item.second);
+    }
 }
 
 void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
@@ -1107,9 +1126,9 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
 
                 if (fReindex) {
                     pblocktree->WriteReindexing(true);
-                    //If we're reindexing in prune mode, wipe away all our block and undo data files
+                    //If we're reindexing in prune mode, wipe away unusable block files and all undo data files
                     if (fPruneMode)
-                        DeleteAllBlockFiles();
+                        CleanupBlockRevFiles();
                 }
 
                 if (!LoadBlockIndex()) {

src/main.cpp

@@ -3057,9 +3057,9 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune)
             if (nCurrentUsage + nBuffer < nPruneTarget)  // are we below our target?
                 break;
 
-            // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip
+            // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning
             if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune)
-                break;
+                continue;
 
             PruneOneBlockFile(fileNumber);
             // Queue up the files for removal