LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

[Wallet] refactor CWallet/CWalletDB/CDB

Browse source 
Try to hide CDB/bitdb behinde CWalletDB.
Prepare for full wallet database abstraction.
This commit is contained in:
Jonas Schnelli 2016-08-24 09:57:23 +02:00
parent fa625b078b
commit 7184e25c80
No known key found for this signature in database
GPG key ID: 1EB776BB03C7922D
5 changed files with 244 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
src/wallet/db.cpp
View file

@ -18,6 +18,7 @@
#endif
#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>
#include <boost/thread.hpp>
#include <boost/version.hpp>
@ -145,7 +146,7 @@ void CDBEnv::MakeMock()
fMockDb = true;
}
CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile))
CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(const std::string& strFile))
{
LOCK(cs_db);
assert(mapFileUseCount.count(strFile) == 0);
@ -158,10 +159,134 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFu
return RECOVER_FAIL;
// Try to recover:
bool fRecovered = (*recoverFunc)(*this, strFile);
bool fRecovered = (*recoverFunc)(strFile);
return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
}
bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue))
{
// Recovery procedure:
// move wallet file to wallet.timestamp.bak
// Call Salvage with fAggressive=true to
// get as much data as possible.
// Rewrite salvaged data to fresh wallet file
// Set -rescan so any missing transactions will be
// found.
int64_t now = GetTime();
std::string newFilename = strprintf("wallet.%d.bak", now);
int result = bitdb.dbenv->dbrename(NULL, filename.c_str(), NULL,
newFilename.c_str(), DB_AUTO_COMMIT);
if (result == 0)
LogPrintf("Renamed %s to %s\n", filename, newFilename);
else
{
LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
return false;
}
std::vector<CDBEnv::KeyValPair> salvagedData;
bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData);
if (salvagedData.empty())
{
LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
return false;
}
LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0));
int ret = pdbCopy->open(NULL, // Txn pointer
filename.c_str(), // Filename
"main", // Logical db name
DB_BTREE, // Database type
DB_CREATE, // Flags
0);
if (ret > 0)
{
LogPrintf("Cannot create database file %s\n", filename);
return false;
}
DbTxn* ptxn = bitdb.TxnBegin();
BOOST_FOREACH(CDBEnv::KeyValPair& row, salvagedData)
{
if (recoverKVcallback)
{
CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
string strType, strErr;
if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
continue;
}
Dbt datKey(&row.first[0], row.first.size());
Dbt datValue(&row.second[0], row.second.size());
int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
if (ret2 > 0)
fSuccess = false;
}
ptxn->commit(0);
pdbCopy->close(0);
return fSuccess;
}
bool CDB::VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr)
{
LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
LogPrintf("Using wallet %s\n", walletFile);
// Wallet file must be a plain filename without a directory
if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile))
{
errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string());
return false;
}
if (!bitdb.Open(dataDir))
{
// try moving the database env out of the way
boost::filesystem::path pathDatabase = dataDir / "database";
boost::filesystem::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime());
try {
boost::filesystem::rename(pathDatabase, pathDatabaseBak);
LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string());
} catch (const boost::filesystem::filesystem_error&) {
// failure is ok (well, not really, but it's not worse than what we started with)
}
// try again
if (!bitdb.Open(dataDir)) {
// if it still fails, it probably means we can't even create the database env
errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir());
return false;
}
}
return true;
}
bool CDB::VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile))
{
if (boost::filesystem::exists(dataDir / walletFile))
{
CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc);
if (r == CDBEnv::RECOVER_OK)
{
warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!"
" Original %s saved as %s in %s; if"
" your balance or transactions are incorrect you should"
" restore from a backup."),
walletFile, "wallet.{timestamp}.bak", dataDir);
}
if (r == CDBEnv::RECOVER_FAIL)
{
errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
return false;
}
}
// also return true if files does not exists
return true;
}
/* End of headers, beginning of key/value data */
static const char *HEADER_END = "HEADER=END";
/* End of key/value data */
@ -473,3 +598,41 @@ void CDBEnv::Flush(bool fShutdown)
}
}
}
bool CDB::PeriodicFlush(std::string strFile)
{
bool ret = false;
TRY_LOCK(bitdb.cs_db,lockDb);
if (lockDb)
{
// Don't do this if any databases are in use
int nRefCount = 0;
map<string, int>::iterator mi = bitdb.mapFileUseCount.begin();
while (mi != bitdb.mapFileUseCount.end())
{
nRefCount += (*mi).second;
mi++;
}
if (nRefCount == 0)
{
boost::this_thread::interruption_point();
map<string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile);
if (mi != bitdb.mapFileUseCount.end())
{
LogPrint("db", "Flushing %s\n", strFile);
int64_t nStart = GetTimeMillis();
// Flush wallet file so it's self contained
bitdb.CloseDb(strFile);
bitdb.CheckpointLSN(strFile);
bitdb.mapFileUseCount.erase(mi++);
LogPrint("db", "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
ret = true;
}
}
}
return ret;
}

11
src/wallet/db.h
View file

@ -56,7 +56,7 @@ public:
enum VerifyResult { VERIFY_OK,
RECOVER_OK,
RECOVER_FAIL };
VerifyResult Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile));
VerifyResult Verify(const std::string& strFile, bool (*recoverFunc)(const std::string& strFile));
/**
* Salvage data from a file that Verify says is bad.
* fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method documentation).
@ -104,6 +104,15 @@ protected:
public:
void Flush();
void Close();
static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue));
/* flush the wallet passively (TRY_LOCK)
ideal to be called periodically */
static bool PeriodicFlush(std::string strFile);
/* verifies the database environment */
static bool VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr);
/* verifies the database file */
static bool VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile));
private:
CDB(const CDB&);

55
src/wallet/wallet.cpp
View file

@ -444,57 +444,30 @@ bool CWallet::Verify()
if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET))
return true;
LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
uiInterface.InitMessage(_("Verifying wallet..."));
std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT);
LogPrintf("Using wallet %s\n", walletFile);
uiInterface.InitMessage(_("Verifying wallet..."));
std::string strError;
if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError))
return InitError(strError);
// Wallet file must be a plain filename without a directory
if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile))
return InitError(strprintf(_("Wallet %s resides outside data directory %s"), walletFile, GetDataDir().string()));
if (!bitdb.Open(GetDataDir()))
{
// try moving the database env out of the way
boost::filesystem::path pathDatabase = GetDataDir() / "database";
boost::filesystem::path pathDatabaseBak = GetDataDir() / strprintf("database.%d.bak", GetTime());
try {
boost::filesystem::rename(pathDatabase, pathDatabaseBak);
LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string());
} catch (const boost::filesystem::filesystem_error&) {
// failure is ok (well, not really, but it's not worse than what we started with)
}
// try again
if (!bitdb.Open(GetDataDir())) {
// if it still fails, it probably means we can't even create the database env
return InitError(strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()));
}
}
if (GetBoolArg("-salvagewallet", false))
{
// Recover readable keypairs:
if (!CWalletDB::Recover(bitdb, walletFile, true))
CWallet dummyWallet;
if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter))
return false;
}
if (boost::filesystem::exists(GetDataDir() / walletFile))
std::string strWarning;
bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError);
if (!strWarning.empty())
InitWarning(strWarning);
if (!dbV)
{
CDBEnv::VerifyResult r = bitdb.Verify(walletFile, CWalletDB::Recover);
if (r == CDBEnv::RECOVER_OK)
{
InitWarning(strprintf(_("Warning: Wallet file corrupt, data salvaged!"
" Original %s saved as %s in %s; if"
" your balance or transactions are incorrect you should"
" restore from a backup."),
walletFile, "wallet.{timestamp}.bak", GetDataDir()));
}
if (r == CDBEnv::RECOVER_FAIL)
return InitError(strprintf(_("%s corrupt, salvage failed"), walletFile));
InitError(strError);
return false;
}
return true;
}

156
src/wallet/walletdb.cpp
View file

@ -546,7 +546,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue,
return true;
}
static bool IsKeyType(string strType)
bool CWalletDB::IsKeyType(const std::string& strType)
{
return (strType== "key" || strType == "wkey" ||
strType == "mkey" || strType == "ckey");
@ -804,38 +804,9 @@ void ThreadFlushWalletDB()
if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2)
{
TRY_LOCK(bitdb.cs_db,lockDb);
if (lockDb)
{
// Don't do this if any databases are in use
int nRefCount = 0;
map<string, int>::iterator mi = bitdb.mapFileUseCount.begin();
while (mi != bitdb.mapFileUseCount.end())
{
nRefCount += (*mi).second;
mi++;
}
if (nRefCount == 0)
{
boost::this_thread::interruption_point();
const std::string& strFile = pwalletMain->strWalletFile;
map<string, int>::iterator _mi = bitdb.mapFileUseCount.find(strFile);
if (_mi != bitdb.mapFileUseCount.end())
{
LogPrint("db", "Flushing %s\n", strFile);
nLastFlushed = CWalletDB::GetUpdateCounter();
int64_t nStart = GetTimeMillis();
// Flush wallet file so it's self contained
bitdb.CloseDb(strFile);
bitdb.CheckpointLSN(strFile);
bitdb.mapFileUseCount.erase(_mi++);
LogPrint("db", "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
}
}
}
const std::string& strFile = pwalletMain->strWalletFile;
if (CDB::PeriodicFlush(strFile))
nLastFlushed = CWalletDB::GetUpdateCounter();
}
}
}
@ -843,90 +814,49 @@ void ThreadFlushWalletDB()
//
// Try to (very carefully!) recover wallet file if there is a problem.
//
bool CWalletDB::Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys)
bool CWalletDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue))
{
// Recovery procedure:
// move wallet file to wallet.timestamp.bak
// Call Salvage with fAggressive=true to
// get as much data as possible.
// Rewrite salvaged data to fresh wallet file
// Set -rescan so any missing transactions will be
// found.
int64_t now = GetTime();
std::string newFilename = strprintf("wallet.%d.bak", now);
int result = dbenv.dbenv->dbrename(NULL, filename.c_str(), NULL,
newFilename.c_str(), DB_AUTO_COMMIT);
if (result == 0)
LogPrintf("Renamed %s to %s\n", filename, newFilename);
else
{
LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
return false;
}
std::vector<CDBEnv::KeyValPair> salvagedData;
bool fSuccess = dbenv.Salvage(newFilename, true, salvagedData);
if (salvagedData.empty())
{
LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
return false;
}
LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
std::unique_ptr<Db> pdbCopy(new Db(dbenv.dbenv, 0));
int ret = pdbCopy->open(NULL, // Txn pointer
filename.c_str(), // Filename
"main", // Logical db name
DB_BTREE, // Database type
DB_CREATE, // Flags
0);
if (ret > 0)
{
LogPrintf("Cannot create database file %s\n", filename);
return false;
}
CWallet dummyWallet;
CWalletScanState wss;
DbTxn* ptxn = dbenv.TxnBegin();
BOOST_FOREACH(CDBEnv::KeyValPair& row, salvagedData)
{
if (fOnlyKeys)
{
CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
string strType, strErr;
bool fReadOK;
{
// Required in LoadKeyMetadata():
LOCK(dummyWallet.cs_wallet);
fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue,
wss, strType, strErr);
}
if (!IsKeyType(strType) && strType != "hdchain")
continue;
if (!fReadOK)
{
LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", strType, strErr);
continue;
}
}
Dbt datKey(&row.first[0], row.first.size());
Dbt datValue(&row.second[0], row.second.size());
int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
if (ret2 > 0)
fSuccess = false;
}
ptxn->commit(0);
pdbCopy->close(0);
return fSuccess;
return CDB::Recover(filename, callbackDataIn, recoverKVcallback);
}
bool CWalletDB::Recover(CDBEnv& dbenv, const std::string& filename)
bool CWalletDB::Recover(const std::string& filename)
{
return CWalletDB::Recover(dbenv, filename, false);
// recover without a key filter callback
// results in recovering all record types
return CWalletDB::Recover(filename, NULL, NULL);
}
bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue)
{
CWallet *dummyWallet = reinterpret_cast<CWallet*>(callbackData);
CWalletScanState dummyWss;
std::string strType, strErr;
bool fReadOK;
{
// Required in LoadKeyMetadata():
LOCK(dummyWallet->cs_wallet);
fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue,
dummyWss, strType, strErr);
}
if (!IsKeyType(strType) && strType != "hdchain")
return false;
if (!fReadOK)
{
LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", strType, strErr);
return false;
}
return true;
}
bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr)
{
return CDB::VerifyEnvironment(walletFile, dataDir, errorStr);
}
bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr)
{
return CDB::VerifyDatabaseFile(walletFile, dataDir, errorStr, warningStr, CWalletDB::Recover);
}
bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value)

14
src/wallet/walletdb.h
View file

@ -170,8 +170,18 @@ public:
DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx);
DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx);
DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut);
static bool Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys);
static bool Recover(CDBEnv& dbenv, const std::string& filename);
/* Try to (very carefully!) recover wallet database (with a possible key type filter) */
static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue));
/* Recover convenience-function to bypass the key filter callback, called when verify failes, recoveres everything */
static bool Recover(const std::string& filename);
/* Recover filter (used as callback), will only let keys (cryptographical keys) as KV/key-type pass through */
static bool RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue);
/* Function to determin if a certain KV/key-type is a key (cryptographical key) type */
static bool IsKeyType(const std::string& strType);
/* verifies the database environment */
static bool VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr);
/* verifies the database file */
static bool VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr);
//! write the hdchain model (external chain child index counter)
bool WriteHDChain(const CHDChain& chain);