Merge pull request #1943 from sipa/reindex2

Add -reindex, to perform in-place reindexing of block chain files
This commit is contained in:
Pieter Wuille 2012-11-09 14:50:30 -08:00
commit 485cf044ba
12 changed files with 408 additions and 151 deletions

View file

@ -293,6 +293,7 @@ std::string HelpMessage()
" -checkblocks=<n> " + _("How many blocks to check at startup (default: 2500, 0 = all)") + "\n" + " -checkblocks=<n> " + _("How many blocks to check at startup (default: 2500, 0 = all)") + "\n" +
" -checklevel=<n> " + _("How thorough the block verification is (0-6, default: 1)") + "\n" + " -checklevel=<n> " + _("How thorough the block verification is (0-6, default: 1)") + "\n" +
" -loadblock=<file> " + _("Imports blocks from external blk000?.dat file") + "\n" + " -loadblock=<file> " + _("Imports blocks from external blk000?.dat file") + "\n" +
" -reindex " + _("Rebuild blockchain index from current blk000??.dat files") + "\n" +
"\n" + _("Block creation options:") + "\n" + "\n" + _("Block creation options:") + "\n" +
" -blockminsize=<n> " + _("Set minimum block size in bytes (default: 0)") + "\n" + " -blockminsize=<n> " + _("Set minimum block size in bytes (default: 0)") + "\n" +
@ -309,6 +310,82 @@ std::string HelpMessage()
} }
struct CImportingNow
{
CImportingNow() {
assert(fImporting == false);
fImporting = true;
}
~CImportingNow() {
assert(fImporting == true);
fImporting = false;
}
};
struct CImportData {
std::vector<boost::filesystem::path> vFiles;
};
void ThreadImport(void *data) {
CImportData *import = reinterpret_cast<CImportData*>(data);
RenameThread("bitcoin-loadblk");
vnThreadsRunning[THREAD_IMPORT]++;
// -reindex
if (fReindex) {
CImportingNow imp;
int nFile = 0;
while (!fShutdown) {
CDiskBlockPos pos;
pos.nFile = nFile;
pos.nPos = 0;
FILE *file = OpenBlockFile(pos, true);
if (!file)
break;
printf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile);
LoadExternalBlockFile(file, &pos);
nFile++;
}
if (!fShutdown) {
pblocktree->WriteReindexing(false);
fReindex = false;
printf("Reindexing finished\n");
}
}
// hardcoded $DATADIR/bootstrap.dat
filesystem::path pathBootstrap = GetDataDir() / "bootstrap.dat";
if (filesystem::exists(pathBootstrap) && !fShutdown) {
FILE *file = fopen(pathBootstrap.string().c_str(), "rb");
if (file) {
CImportingNow imp;
filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old";
printf("Importing bootstrap.dat...\n");
LoadExternalBlockFile(file);
RenameOver(pathBootstrap, pathBootstrapOld);
}
}
// -loadblock=
BOOST_FOREACH(boost::filesystem::path &path, import->vFiles) {
if (fShutdown)
break;
FILE *file = fopen(path.string().c_str(), "rb");
if (file) {
CImportingNow imp;
printf("Importing %s...\n", path.string().c_str());
LoadExternalBlockFile(file);
}
}
delete import;
vnThreadsRunning[THREAD_IMPORT]--;
}
/** Initialize bitcoin. /** Initialize bitcoin.
* @pre Parameters should be parsed and config file should be read. * @pre Parameters should be parsed and config file should be read.
*/ */
@ -639,6 +716,8 @@ bool AppInit2()
// ********************************************************* Step 7: load block chain // ********************************************************* Step 7: load block chain
fReindex = GetBoolArg("-reindex");
if (!bitdb.Open(GetDataDir())) if (!bitdb.Open(GetDataDir()))
{ {
string msg = strprintf(_("Error initializing database environment %s!" string msg = strprintf(_("Error initializing database environment %s!"
@ -662,10 +741,13 @@ bool AppInit2()
uiInterface.InitMessage(_("Loading block index...")); uiInterface.InitMessage(_("Loading block index..."));
printf("Loading block index...\n"); printf("Loading block index...\n");
nStart = GetTimeMillis(); nStart = GetTimeMillis();
pblocktree = new CBlockTreeDB(nBlockTreeDBCache); pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex);
pcoinsdbview = new CCoinsViewDB(nCoinDBCache); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
pcoinsTip = new CCoinsViewCache(*pcoinsdbview); pcoinsTip = new CCoinsViewCache(*pcoinsdbview);
if (fReindex)
pblocktree->WriteReindexing(true);
if (!LoadBlockIndex()) if (!LoadBlockIndex())
return InitError(_("Error loading blkindex.dat")); return InitError(_("Error loading blkindex.dat"));
@ -798,13 +880,13 @@ bool AppInit2()
if (!ConnectBestBlock()) if (!ConnectBestBlock())
strErrors << "Failed to connect best block"; strErrors << "Failed to connect best block";
std::vector<boost::filesystem::path> *vPath = new std::vector<boost::filesystem::path>(); CImportData *pimport = new CImportData();
if (mapArgs.count("-loadblock")) if (mapArgs.count("-loadblock"))
{ {
BOOST_FOREACH(string strFile, mapMultiArgs["-loadblock"]) BOOST_FOREACH(string strFile, mapMultiArgs["-loadblock"])
vPath->push_back(strFile); pimport->vFiles.push_back(strFile);
} }
NewThread(ThreadImport, vPath); NewThread(ThreadImport, pimport);
// ********************************************************* Step 10: load peers // ********************************************************* Step 10: load peers

View file

@ -21,7 +21,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) {
return options; return options;
} }
CLevelDB::CLevelDB(const boost::filesystem::path &path, size_t nCacheSize, bool fMemory) { CLevelDB::CLevelDB(const boost::filesystem::path &path, size_t nCacheSize, bool fMemory, bool fWipe) {
penv = NULL; penv = NULL;
readoptions.verify_checksums = true; readoptions.verify_checksums = true;
iteroptions.verify_checksums = true; iteroptions.verify_checksums = true;
@ -33,6 +33,10 @@ CLevelDB::CLevelDB(const boost::filesystem::path &path, size_t nCacheSize, bool
penv = leveldb::NewMemEnv(leveldb::Env::Default()); penv = leveldb::NewMemEnv(leveldb::Env::Default());
options.env = penv; options.env = penv;
} else { } else {
if (fWipe) {
printf("Wiping LevelDB in %s\n", path.string().c_str());
leveldb::DestroyDB(path.string(), options);
}
boost::filesystem::create_directory(path); boost::filesystem::create_directory(path);
printf("Opening LevelDB in %s\n", path.string().c_str()); printf("Opening LevelDB in %s\n", path.string().c_str());
} }

View file

@ -69,7 +69,7 @@ private:
leveldb::DB *pdb; leveldb::DB *pdb;
public: public:
CLevelDB(const boost::filesystem::path &path, size_t nCacheSize, bool fMemory = false); CLevelDB(const boost::filesystem::path &path, size_t nCacheSize, bool fMemory = false, bool fWipe = false);
~CLevelDB(); ~CLevelDB();
template<typename K, typename V> bool Read(const K& key, V& value) { template<typename K, typename V> bool Read(const K& key, V& value) {

View file

@ -41,6 +41,7 @@ CBlockIndex* pindexBest = NULL;
set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexValid; // may contain all CBlockIndex*'s that have validness >=BLOCK_VALID_TRANSACTIONS, and must contain those who aren't failed set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexValid; // may contain all CBlockIndex*'s that have validness >=BLOCK_VALID_TRANSACTIONS, and must contain those who aren't failed
int64 nTimeBestReceived = 0; int64 nTimeBestReceived = 0;
bool fImporting = false; bool fImporting = false;
bool fReindex = false;
unsigned int nCoinCacheSize = 5000; unsigned int nCoinCacheSize = 5000;
CMedianFilter<int> cPeerBlockCounts(5, 0); // Amount of blocks that other nodes claim to have CMedianFilter<int> cPeerBlockCounts(5, 0); // Amount of blocks that other nodes claim to have
@ -1145,7 +1146,7 @@ int GetNumBlocksOfPeers()
bool IsInitialBlockDownload() bool IsInitialBlockDownload()
{ {
if (pindexBest == NULL || nBestHeight < Checkpoints::GetTotalBlocksEstimate()) if (pindexBest == NULL || nBestHeight < Checkpoints::GetTotalBlocksEstimate() || fReindex || fImporting)
return true; return true;
static int64 nLastUpdate; static int64 nLastUpdate;
static CBlockIndex* pindexLastBest; static CBlockIndex* pindexLastBest;
@ -1862,35 +1863,45 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos)
} }
bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime, bool fKnown = false)
{ {
bool fUpdatedLast = false; bool fUpdatedLast = false;
LOCK(cs_LastBlockFile); LOCK(cs_LastBlockFile);
while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { if (fKnown) {
printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); if (nLastBlockFile != pos.nFile) {
FlushBlockFile(); nLastBlockFile = pos.nFile;
nLastBlockFile++; infoLastBlockFile.SetNull();
infoLastBlockFile.SetNull(); pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile);
pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine }
fUpdatedLast = true; } else {
while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) {
printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str());
FlushBlockFile();
nLastBlockFile++;
infoLastBlockFile.SetNull();
pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine
fUpdatedLast = true;
}
pos.nFile = nLastBlockFile;
pos.nPos = infoLastBlockFile.nSize;
} }
pos.nFile = nLastBlockFile;
pos.nPos = infoLastBlockFile.nSize;
infoLastBlockFile.nSize += nAddSize; infoLastBlockFile.nSize += nAddSize;
infoLastBlockFile.AddBlock(nHeight, nTime); infoLastBlockFile.AddBlock(nHeight, nTime);
unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; if (!fKnown) {
unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) { unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
FILE *file = OpenBlockFile(pos); if (nNewChunks > nOldChunks) {
if (file) { FILE *file = OpenBlockFile(pos);
printf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); if (file) {
AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); printf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile);
AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos);
}
fclose(file);
} }
fclose(file);
} }
if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile))
@ -1996,7 +2007,7 @@ bool CBlock::CheckBlock(bool fCheckPOW, bool fCheckMerkleRoot) const
return true; return true;
} }
bool CBlock::AcceptBlock() bool CBlock::AcceptBlock(CDiskBlockPos *dbp)
{ {
// Check for duplicate // Check for duplicate
uint256 hash = GetHash(); uint256 hash = GetHash();
@ -2004,11 +2015,15 @@ bool CBlock::AcceptBlock()
return error("AcceptBlock() : block already in mapBlockIndex"); return error("AcceptBlock() : block already in mapBlockIndex");
// Get prev block index // Get prev block index
CBlockIndex* pindexPrev = NULL;
int nHeight = 0;
if (hash != hashGenesisBlock) {
map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hashPrevBlock); map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hashPrevBlock);
if (mi == mapBlockIndex.end()) if (mi == mapBlockIndex.end())
return DoS(10, error("AcceptBlock() : prev block not found")); return DoS(10, error("AcceptBlock() : prev block not found"));
CBlockIndex* pindexPrev = (*mi).second; pindexPrev = (*mi).second;
int nHeight = pindexPrev->nHeight+1; nHeight = pindexPrev->nHeight+1;
// Check proof of work // Check proof of work
if (nBits != GetNextWorkRequired(pindexPrev, this)) if (nBits != GetNextWorkRequired(pindexPrev, this))
@ -2048,16 +2063,22 @@ bool CBlock::AcceptBlock()
return DoS(100, error("AcceptBlock() : block height mismatch in coinbase")); return DoS(100, error("AcceptBlock() : block height mismatch in coinbase"));
} }
} }
}
// Write block to history file // Write block to history file
unsigned int nBlockSize = ::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION); unsigned int nBlockSize = ::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION);
if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION)))
return error("AcceptBlock() : out of disk space");
CDiskBlockPos blockPos; CDiskBlockPos blockPos;
if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, nTime)) if (dbp == NULL) {
if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION)))
return error("AcceptBlock() : out of disk space");
} else {
blockPos = *dbp;
}
if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, nTime, dbp != NULL))
return error("AcceptBlock() : FindBlockPos failed"); return error("AcceptBlock() : FindBlockPos failed");
if (!WriteToDisk(blockPos)) if (dbp == NULL)
return error("AcceptBlock() : WriteToDisk failed"); if (!WriteToDisk(blockPos))
return error("AcceptBlock() : WriteToDisk failed");
if (!AddToBlockIndex(blockPos)) if (!AddToBlockIndex(blockPos))
return error("AcceptBlock() : AddToBlockIndex failed"); return error("AcceptBlock() : AddToBlockIndex failed");
@ -2086,7 +2107,7 @@ bool CBlockIndex::IsSuperMajority(int minVersion, const CBlockIndex* pstart, uns
return (nFound >= nRequired); return (nFound >= nRequired);
} }
bool ProcessBlock(CNode* pfrom, CBlock* pblock) bool ProcessBlock(CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp)
{ {
// Check for duplicate // Check for duplicate
uint256 hash = pblock->GetHash(); uint256 hash = pblock->GetHash();
@ -2124,7 +2145,7 @@ bool ProcessBlock(CNode* pfrom, CBlock* pblock)
// If we don't already have its previous block, shunt it off to holding area until we get it // If we don't already have its previous block, shunt it off to holding area until we get it
if (!mapBlockIndex.count(pblock->hashPrevBlock)) if (pblock->hashPrevBlock != 0 && !mapBlockIndex.count(pblock->hashPrevBlock))
{ {
printf("ProcessBlock: ORPHAN BLOCK, prev=%s\n", pblock->hashPrevBlock.ToString().substr(0,20).c_str()); printf("ProcessBlock: ORPHAN BLOCK, prev=%s\n", pblock->hashPrevBlock.ToString().substr(0,20).c_str());
@ -2141,7 +2162,7 @@ bool ProcessBlock(CNode* pfrom, CBlock* pblock)
} }
// Store to disk // Store to disk
if (!pblock->AcceptBlock()) if (!pblock->AcceptBlock(dbp))
return error("ProcessBlock() : AcceptBlock FAILED"); return error("ProcessBlock() : AcceptBlock FAILED");
// Recursively process any orphan blocks that depended on this one // Recursively process any orphan blocks that depended on this one
@ -2304,6 +2325,11 @@ bool static LoadBlockIndexDB()
// Load bnBestInvalidWork, OK if it doesn't exist // Load bnBestInvalidWork, OK if it doesn't exist
pblocktree->ReadBestInvalidWork(bnBestInvalidWork); pblocktree->ReadBestInvalidWork(bnBestInvalidWork);
// Check whether we need to continue reindexing
bool fReindexing = false;
pblocktree->ReadReindexing(fReindexing);
fReindex |= fReindexing;
// Verify blocks in the best chain // Verify blocks in the best chain
int nCheckLevel = GetArg("-checklevel", 1); int nCheckLevel = GetArg("-checklevel", 1);
int nCheckDepth = GetArg( "-checkblocks", 2500); int nCheckDepth = GetArg( "-checkblocks", 2500);
@ -2337,7 +2363,7 @@ bool static LoadBlockIndexDB()
return true; return true;
} }
bool LoadBlockIndex(bool fAllowNew) bool LoadBlockIndex()
{ {
if (fTestNet) if (fTestNet)
{ {
@ -2348,6 +2374,9 @@ bool LoadBlockIndex(bool fAllowNew)
hashGenesisBlock = uint256("000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"); hashGenesisBlock = uint256("000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943");
} }
if (fReindex)
return true;
// //
// Load block index from databases // Load block index from databases
// //
@ -2359,9 +2388,6 @@ bool LoadBlockIndex(bool fAllowNew)
// //
if (mapBlockIndex.empty()) if (mapBlockIndex.empty())
{ {
if (!fAllowNew)
return false;
// Genesis Block: // Genesis Block:
// CBlock(hash=000000000019d6, ver=1, hashPrevBlock=00000000000000, hashMerkleRoot=4a5e1e, nTime=1231006505, nBits=1d00ffff, nNonce=2083236893, vtx=1) // CBlock(hash=000000000019d6, ver=1, hashPrevBlock=00000000000000, hashMerkleRoot=4a5e1e, nTime=1231006505, nBits=1d00ffff, nNonce=2083236893, vtx=1)
// CTransaction(hash=4a5e1e, ver=1, vin.size=1, vout.size=1, nLockTime=0) // CTransaction(hash=4a5e1e, ver=1, vin.size=1, vout.size=1, nLockTime=0)
@ -2487,110 +2513,71 @@ void PrintBlockTree()
} }
} }
bool LoadExternalBlockFile(FILE* fileIn) bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp)
{ {
int64 nStart = GetTimeMillis(); int64 nStart = GetTimeMillis();
int nLoaded = 0; int nLoaded = 0;
{ {
try { CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SIZE, MAX_BLOCK_SIZE+8, SER_DISK, CLIENT_VERSION);
CAutoFile blkdat(fileIn, SER_DISK, CLIENT_VERSION); uint64 nStartByte = 0;
unsigned int nPos = 0; if (dbp) {
while (nPos != (unsigned int)-1 && blkdat.good() && !fRequestShutdown) // (try to) skip already indexed part
{ CBlockFileInfo info;
unsigned char pchData[65536]; if (pblocktree->ReadBlockFileInfo(dbp->nFile, info)) {
do { nStartByte = info.nSize;
fseek(blkdat, nPos, SEEK_SET); blkdat.Seek(info.nSize);
int nRead = fread(pchData, 1, sizeof(pchData), blkdat);
if (nRead <= 8)
{
nPos = (unsigned int)-1;
break;
}
void* nFind = memchr(pchData, pchMessageStart[0], nRead+1-sizeof(pchMessageStart));
if (nFind)
{
if (memcmp(nFind, pchMessageStart, sizeof(pchMessageStart))==0)
{
nPos += ((unsigned char*)nFind - pchData) + sizeof(pchMessageStart);
break;
}
nPos += ((unsigned char*)nFind - pchData) + 1;
}
else
nPos += sizeof(pchData) - sizeof(pchMessageStart) + 1;
} while(!fRequestShutdown);
if (nPos == (unsigned int)-1)
break;
fseek(blkdat, nPos, SEEK_SET);
unsigned int nSize;
blkdat >> nSize;
if (nSize > 0 && nSize <= MAX_BLOCK_SIZE)
{
CBlock block;
blkdat >> block;
LOCK(cs_main);
if (ProcessBlock(NULL,&block))
{
nLoaded++;
nPos += 4 + nSize;
}
}
} }
} }
catch (std::exception &e) { uint64 nRewind = blkdat.GetPos();
printf("%s() : Deserialize or I/O error caught during load\n", while (blkdat.good() && !blkdat.eof() && !fShutdown) {
__PRETTY_FUNCTION__); blkdat.SetPos(nRewind);
nRewind++; // start one byte further next time, in case of failure
blkdat.SetLimit(); // remove former limit
unsigned int nSize = 0;
try {
// locate a header
unsigned char buf[4];
blkdat.FindByte(pchMessageStart[0]);
nRewind = blkdat.GetPos()+1;
blkdat >> FLATDATA(buf);
if (memcmp(buf, pchMessageStart, 4))
continue;
// read size
blkdat >> nSize;
if (nSize < 80 || nSize > MAX_BLOCK_SIZE)
continue;
} catch (std::exception &e) {
// no valid block header found; don't complain
break;
}
try {
// read block
uint64 nBlockPos = blkdat.GetPos();
blkdat.SetLimit(nBlockPos + nSize);
CBlock block;
blkdat >> block;
nRewind = blkdat.GetPos();
// process block
if (nBlockPos >= nStartByte) {
LOCK(cs_main);
if (dbp)
dbp->nPos = nBlockPos;
if (ProcessBlock(NULL, &block, dbp))
nLoaded++;
}
} catch (std::exception &e) {
printf("%s() : Deserialize or I/O error caught during load\n", __PRETTY_FUNCTION__);
}
} }
fclose(fileIn);
} }
printf("Loaded %i blocks from external file in %"PRI64d"ms\n", nLoaded, GetTimeMillis() - nStart); if (nLoaded > 0)
printf("Loaded %i blocks from external file in %"PRI64d"ms\n", nLoaded, GetTimeMillis() - nStart);
return nLoaded > 0; return nLoaded > 0;
} }
struct CImportingNow
{
CImportingNow() {
assert(fImporting == false);
fImporting = true;
}
~CImportingNow() {
assert(fImporting == true);
fImporting = false;
}
};
void ThreadImport(void *data) {
std::vector<boost::filesystem::path> *vFiles = reinterpret_cast<std::vector<boost::filesystem::path>*>(data);
RenameThread("bitcoin-loadblk");
CImportingNow imp;
vnThreadsRunning[THREAD_IMPORT]++;
// -loadblock=
BOOST_FOREACH(boost::filesystem::path &path, *vFiles) {
FILE *file = fopen(path.string().c_str(), "rb");
if (file)
LoadExternalBlockFile(file);
}
// hardcoded $DATADIR/bootstrap.dat
filesystem::path pathBootstrap = GetDataDir() / "bootstrap.dat";
if (filesystem::exists(pathBootstrap)) {
FILE *file = fopen(pathBootstrap.string().c_str(), "rb");
if (file) {
filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old";
LoadExternalBlockFile(file);
RenameOver(pathBootstrap, pathBootstrapOld);
}
}
delete vFiles;
vnThreadsRunning[THREAD_IMPORT]--;
}
@ -2800,7 +2787,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
// Ask the first connected node for block updates // Ask the first connected node for block updates
static int nAskedForBlocks = 0; static int nAskedForBlocks = 0;
if (!pfrom->fClient && !pfrom->fOneShot && !fImporting && if (!pfrom->fClient && !pfrom->fOneShot && !fImporting && !fReindex &&
(pfrom->nStartingHeight > (nBestHeight - 144)) && (pfrom->nStartingHeight > (nBestHeight - 144)) &&
(pfrom->nVersion < NOBLKS_VERSION_START || (pfrom->nVersion < NOBLKS_VERSION_START ||
pfrom->nVersion >= NOBLKS_VERSION_END) && pfrom->nVersion >= NOBLKS_VERSION_END) &&
@ -2937,7 +2924,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
printf(" got inventory: %s %s\n", inv.ToString().c_str(), fAlreadyHave ? "have" : "new"); printf(" got inventory: %s %s\n", inv.ToString().c_str(), fAlreadyHave ? "have" : "new");
if (!fAlreadyHave) { if (!fAlreadyHave) {
if (!fImporting) if (!fImporting && !fReindex)
pfrom->AskFor(inv); pfrom->AskFor(inv);
} else if (inv.type == MSG_BLOCK && mapOrphanBlocks.count(inv.hash)) { } else if (inv.type == MSG_BLOCK && mapOrphanBlocks.count(inv.hash)) {
pfrom->PushGetBlocks(pindexBest, GetOrphanRoot(mapOrphanBlocks[inv.hash])); pfrom->PushGetBlocks(pindexBest, GetOrphanRoot(mapOrphanBlocks[inv.hash]));
@ -3167,7 +3154,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
} }
else if (strCommand == "block") else if (strCommand == "block" && !fImporting && !fReindex) // Ignore blocks received while importing
{ {
CBlock block; CBlock block;
vRecv >> block; vRecv >> block;

View file

@ -88,6 +88,7 @@ extern CCriticalSection cs_setpwalletRegistered;
extern std::set<CWallet*> setpwalletRegistered; extern std::set<CWallet*> setpwalletRegistered;
extern unsigned char pchMessageStart[4]; extern unsigned char pchMessageStart[4];
extern bool fImporting; extern bool fImporting;
extern bool fReindex;
extern unsigned int nCoinCacheSize; extern unsigned int nCoinCacheSize;
// Settings // Settings
@ -109,11 +110,12 @@ class CCoinsViewCache;
void RegisterWallet(CWallet* pwalletIn); void RegisterWallet(CWallet* pwalletIn);
void UnregisterWallet(CWallet* pwalletIn); void UnregisterWallet(CWallet* pwalletIn);
void SyncWithWallets(const uint256 &hash, const CTransaction& tx, const CBlock* pblock = NULL, bool fUpdate = false); void SyncWithWallets(const uint256 &hash, const CTransaction& tx, const CBlock* pblock = NULL, bool fUpdate = false);
bool ProcessBlock(CNode* pfrom, CBlock* pblock); bool ProcessBlock(CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp = NULL);
bool CheckDiskSpace(uint64 nAdditionalBytes=0); bool CheckDiskSpace(uint64 nAdditionalBytes=0);
FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false);
FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false);
bool LoadBlockIndex(bool fAllowNew=true); bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp = NULL);
bool LoadBlockIndex();
void PrintBlockTree(); void PrintBlockTree();
CBlockIndex* FindBlockByHeight(int nHeight); CBlockIndex* FindBlockByHeight(int nHeight);
bool ProcessMessages(CNode* pfrom); bool ProcessMessages(CNode* pfrom);
@ -1261,7 +1263,8 @@ public:
bool CheckBlock(bool fCheckPOW=true, bool fCheckMerkleRoot=true) const; bool CheckBlock(bool fCheckPOW=true, bool fCheckMerkleRoot=true) const;
// Store block on disk // Store block on disk
bool AcceptBlock(); // if dbp is provided, the file is known to already reside on disk
bool AcceptBlock(CDiskBlockPos *dbp = NULL);
}; };

View file

@ -489,7 +489,8 @@ void BitcoinGUI::setNumBlocks(int count, int nTotalBlocks)
statusBar()->clearMessage(); statusBar()->clearMessage();
// don't show / hide progress bar and its label if we have no connection to the network // don't show / hide progress bar and its label if we have no connection to the network
if (!clientModel || (clientModel->getNumConnections() == 0 && !clientModel->isImporting())) enum BlockSource blockSource = clientModel ? clientModel->getBlockSource() : BLOCK_SOURCE_NONE;
if (blockSource == BLOCK_SOURCE_NONE || (blockSource == BLOCK_SOURCE_NETWORK && clientModel->getNumConnections() == 0))
{ {
progressBarLabel->setVisible(false); progressBarLabel->setVisible(false);
progressBar->setVisible(false); progressBar->setVisible(false);
@ -499,26 +500,37 @@ void BitcoinGUI::setNumBlocks(int count, int nTotalBlocks)
QString tooltip; QString tooltip;
QString importText;
switch (blockSource) {
case BLOCK_SOURCE_NONE:
case BLOCK_SOURCE_NETWORK:
importText = tr("Synchronizing with network...");
case BLOCK_SOURCE_DISK:
importText = tr("Importing blocks from disk...");
case BLOCK_SOURCE_REINDEX:
importText = tr("Reindexing blocks on disk...");
}
if(count < nTotalBlocks) if(count < nTotalBlocks)
{ {
int nRemainingBlocks = nTotalBlocks - count; int nRemainingBlocks = nTotalBlocks - count;
float nPercentageDone = count / (nTotalBlocks * 0.01f); float nPercentageDone = count / (nTotalBlocks * 0.01f);
progressBarLabel->setText(tr(clientModel->isImporting() ? "Importing blocks..." : "Synchronizing with network...")); progressBarLabel->setText(importText);
progressBarLabel->setVisible(true); progressBarLabel->setVisible(true);
progressBar->setFormat(tr("~%n block(s) remaining", "", nRemainingBlocks)); progressBar->setFormat(tr("~%n block(s) remaining", "", nRemainingBlocks));
progressBar->setMaximum(nTotalBlocks); progressBar->setMaximum(nTotalBlocks);
progressBar->setValue(count); progressBar->setValue(count);
progressBar->setVisible(true); progressBar->setVisible(true);
tooltip = tr("Downloaded %1 of %2 blocks of transaction history (%3% done).").arg(count).arg(nTotalBlocks).arg(nPercentageDone, 0, 'f', 2); tooltip = tr("Processed %1 of %2 blocks of transaction history (%3% done).").arg(count).arg(nTotalBlocks).arg(nPercentageDone, 0, 'f', 2);
} }
else else
{ {
progressBarLabel->setVisible(false); progressBarLabel->setVisible(false);
progressBar->setVisible(false); progressBar->setVisible(false);
tooltip = tr("Downloaded %1 blocks of transaction history.").arg(count); tooltip = tr("Processed %1 blocks of transaction history.").arg(count);
} }
QDateTime lastBlockDate = clientModel->getLastBlockDate(); QDateTime lastBlockDate = clientModel->getLastBlockDate();

View file

@ -101,9 +101,13 @@ bool ClientModel::inInitialBlockDownload() const
return IsInitialBlockDownload(); return IsInitialBlockDownload();
} }
bool ClientModel::isImporting() const enum BlockSource ClientModel::getBlockSource() const
{ {
return fImporting; if (fReindex)
return BLOCK_SOURCE_REINDEX;
if (fImporting)
return BLOCK_SOURCE_DISK;
return BLOCK_SOURCE_NETWORK;
} }
int ClientModel::getNumBlocksOfPeers() const int ClientModel::getNumBlocksOfPeers() const

View file

@ -13,6 +13,13 @@ class QDateTime;
class QTimer; class QTimer;
QT_END_NAMESPACE QT_END_NAMESPACE
enum BlockSource {
BLOCK_SOURCE_NONE,
BLOCK_SOURCE_NETWORK,
BLOCK_SOURCE_DISK,
BLOCK_SOURCE_REINDEX
};
/** Model for Bitcoin network client. */ /** Model for Bitcoin network client. */
class ClientModel : public QObject class ClientModel : public QObject
{ {
@ -34,7 +41,7 @@ public:
//! Return true if core is doing initial block download //! Return true if core is doing initial block download
bool inInitialBlockDownload() const; bool inInitialBlockDownload() const;
//! Return true if core is importing blocks //! Return true if core is importing blocks
bool isImporting() const; enum BlockSource getBlockSource() const;
//! Return conservative estimate of total number of blocks, or 0 if unknown //! Return conservative estimate of total number of blocks, or 0 if unknown
int getNumBlocksOfPeers() const; int getNumBlocksOfPeers() const;
//! Return warnings to be displayed in status bar //! Return warnings to be displayed in status bar

View file

@ -1225,4 +1225,148 @@ public:
} }
}; };
/** Wrapper around a FILE* that implements a ring buffer to
* deserialize from. It guarantees the ability to rewind
* a given number of bytes. */
class CBufferedFile
{
private:
FILE *src; // source file
uint64 nSrcPos; // how many bytes have been read from source
uint64 nReadPos; // how many bytes have been read from this
uint64 nReadLimit; // up to which position we're allowed to read
uint64 nRewind; // how many bytes we guarantee to rewind
std::vector<char> vchBuf; // the buffer
short state;
short exceptmask;
protected:
void setstate(short bits, const char *psz) {
state |= bits;
if (state & exceptmask)
throw std::ios_base::failure(psz);
}
// read data from the source to fill the buffer
bool Fill() {
unsigned int pos = nSrcPos % vchBuf.size();
unsigned int readNow = vchBuf.size() - pos;
unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
if (nAvail < readNow)
readNow = nAvail;
if (readNow == 0)
return false;
size_t read = fread((void*)&vchBuf[pos], 1, readNow, src);
if (read == 0) {
setstate(std::ios_base::failbit, feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed");
return false;
} else {
nSrcPos += read;
return true;
}
}
public:
int nType;
int nVersion;
CBufferedFile(FILE *fileIn, uint64 nBufSize, uint64 nRewindIn, int nTypeIn, int nVersionIn) :
src(fileIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0),
state(0), exceptmask(std::ios_base::badbit | std::ios_base::failbit), nType(nTypeIn), nVersion(nVersionIn) {
}
// check whether no error occurred
bool good() const {
return state == 0;
}
// check whether we're at the end of the source file
bool eof() const {
return nReadPos == nSrcPos && feof(src);
}
// read a number of bytes
CBufferedFile& read(char *pch, size_t nSize) {
if (nSize + nReadPos > nReadLimit)
throw std::ios_base::failure("Read attempted past buffer limit");
if (nSize + nRewind > vchBuf.size())
throw std::ios_base::failure("Read larger than buffer size");
while (nSize > 0) {
if (nReadPos == nSrcPos)
Fill();
unsigned int pos = nReadPos % vchBuf.size();
size_t nNow = nSize;
if (nNow + pos > vchBuf.size())
nNow = vchBuf.size() - pos;
if (nNow + nReadPos > nSrcPos)
nNow = nSrcPos - nReadPos;
memcpy(pch, &vchBuf[pos], nNow);
nReadPos += nNow;
pch += nNow;
nSize -= nNow;
}
return (*this);
}
// return the current reading position
uint64 GetPos() {
return nReadPos;
}
// rewind to a given reading position
bool SetPos(uint64 nPos) {
nReadPos = nPos;
if (nReadPos + nRewind < nSrcPos) {
nReadPos = nSrcPos - nRewind;
return false;
} else if (nReadPos > nSrcPos) {
nReadPos = nSrcPos;
return false;
} else {
return true;
}
}
bool Seek(uint64 nPos) {
long nLongPos = nPos;
if (nPos != (uint64)nLongPos)
return false;
if (fseek(src, nLongPos, SEEK_SET))
return false;
nLongPos = ftell(src);
nSrcPos = nLongPos;
nReadPos = nLongPos;
state = 0;
return true;
}
// prevent reading beyond a certain position
// no argument removes the limit
bool SetLimit(uint64 nPos = (uint64)(-1)) {
if (nPos < nReadPos)
return false;
nReadLimit = nPos;
return true;
}
template<typename T>
CBufferedFile& operator>>(T& obj) {
// Unserialize from this stream
::Unserialize(*this, obj, nType, nVersion);
return (*this);
}
// search for a given byte in the stream, and remain positioned on it
void FindByte(char ch) {
while (true) {
if (nReadPos == nSrcPos)
Fill();
if (vchBuf[nReadPos % vchBuf.size()] == ch)
break;
nReadPos++;
}
}
};
#endif #endif

View file

@ -22,7 +22,7 @@ struct TestingSetup {
pblocktree = new CBlockTreeDB(true); pblocktree = new CBlockTreeDB(true);
pcoinsdbview = new CCoinsViewDB(true); pcoinsdbview = new CCoinsViewDB(true);
pcoinsTip = new CCoinsViewCache(*pcoinsdbview); pcoinsTip = new CCoinsViewCache(*pcoinsdbview);
LoadBlockIndex(true); LoadBlockIndex();
bool fFirstRun; bool fFirstRun;
pwalletMain = new CWallet("wallet.dat"); pwalletMain = new CWallet("wallet.dat");
pwalletMain->LoadWallet(fFirstRun); pwalletMain->LoadWallet(fFirstRun);

View file

@ -19,7 +19,7 @@ void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) {
batch.Write('B', hash); batch.Write('B', hash);
} }
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory) : db(GetDataDir() / "coins", nCacheSize, fMemory) { CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "coins", nCacheSize, fMemory, fWipe) {
} }
bool CCoinsViewDB::GetCoins(uint256 txid, CCoins &coins) { bool CCoinsViewDB::GetCoins(uint256 txid, CCoins &coins) {
@ -64,7 +64,7 @@ bool CCoinsViewDB::BatchWrite(const std::map<uint256, CCoins> &mapCoins, CBlockI
return db.WriteBatch(batch); return db.WriteBatch(batch);
} }
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory) : CLevelDB(GetDataDir() / "blktree", nCacheSize, fMemory) { CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CLevelDB(GetDataDir() / "blktree", nCacheSize, fMemory, fWipe) {
} }
bool CBlockTreeDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) bool CBlockTreeDB::WriteBlockIndex(const CDiskBlockIndex& blockindex)
@ -94,6 +94,18 @@ bool CBlockTreeDB::WriteLastBlockFile(int nFile) {
return Write('l', nFile); return Write('l', nFile);
} }
bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
if (fReindexing)
return Write('R', '1');
else
return Erase('R');
}
bool CBlockTreeDB::ReadReindexing(bool &fReindexing) {
fReindexing = Exists('R');
return true;
}
bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { bool CBlockTreeDB::ReadLastBlockFile(int &nFile) {
return Read('l', nFile); return Read('l', nFile);
} }

View file

@ -14,7 +14,7 @@ class CCoinsViewDB : public CCoinsView
protected: protected:
CLevelDB db; CLevelDB db;
public: public:
CCoinsViewDB(size_t nCacheSize, bool fMemory = false); CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
bool GetCoins(uint256 txid, CCoins &coins); bool GetCoins(uint256 txid, CCoins &coins);
bool SetCoins(uint256 txid, const CCoins &coins); bool SetCoins(uint256 txid, const CCoins &coins);
@ -29,7 +29,7 @@ public:
class CBlockTreeDB : public CLevelDB class CBlockTreeDB : public CLevelDB
{ {
public: public:
CBlockTreeDB(size_t nCacheSize, bool fMemory = false); CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
private: private:
CBlockTreeDB(const CBlockTreeDB&); CBlockTreeDB(const CBlockTreeDB&);
void operator=(const CBlockTreeDB&); void operator=(const CBlockTreeDB&);
@ -41,6 +41,8 @@ public:
bool WriteBlockFileInfo(int nFile, const CBlockFileInfo &fileinfo); bool WriteBlockFileInfo(int nFile, const CBlockFileInfo &fileinfo);
bool ReadLastBlockFile(int &nFile); bool ReadLastBlockFile(int &nFile);
bool WriteLastBlockFile(int nFile); bool WriteLastBlockFile(int nFile);
bool WriteReindexing(bool fReindex);
bool ReadReindexing(bool &fReindex);
bool LoadBlockIndexGuts(); bool LoadBlockIndexGuts();
}; };