LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

index: Move index DBs into index/ directory.

Browse source
This commit is contained in:
Jim Posen 2018-05-15 17:20:17 -07:00
parent 89eddcd365
commit ec3073a274
6 changed files with 264 additions and 245 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/index/base.cpp
View file

@ -11,6 +11,8 @@
#include <validation.h>
#include <warnings.h>
constexpr char DB_BEST_BLOCK = 'B';
constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
@ -26,6 +28,24 @@ static void FatalError(const char* fmt, const Args&... args)
StartShutdown();
}
BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) :
CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate)
{}
bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
{
bool success = Read(DB_BEST_BLOCK, locator);
if (!success) {
locator.SetNull();
}
return success;
}
bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
{
return Write(DB_BEST_BLOCK, locator);
}
BaseIndex::~BaseIndex()
{
Interrupt();

18
src/index/base.h
View file

@ -5,10 +5,10 @@
#ifndef BITCOIN_INDEX_BASE_H
#define BITCOIN_INDEX_BASE_H
#include <dbwrapper.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <threadinterrupt.h>
#include <txdb.h>
#include <uint256.h>
#include <validationinterface.h>
@ -21,6 +21,20 @@ class CBlockIndex;
*/
class BaseIndex : public CValidationInterface
{
protected:
class DB : public CDBWrapper
{
public:
DB(const fs::path& path, size_t n_cache_size,
bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
/// Read block locator of the chain that the txindex is in sync with.
bool ReadBestBlock(CBlockLocator& locator) const;
/// Write block locator of the chain that the txindex is in sync with.
bool WriteBestBlock(const CBlockLocator& locator);
};
private:
/// Whether the index is in sync with the main chain. The flag is flipped
/// from false to true once, after which point this starts processing
@ -55,7 +69,7 @@ protected:
/// Write update index entries for a newly connected block.
virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
virtual BaseIndexDB& GetDB() const = 0;
virtual DB& GetDB() const = 0;
/// Get the name of the index for display in logs.
virtual const char* GetName() const = 0;

219
src/index/txindex.cpp
View file

@ -3,15 +3,232 @@
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <index/txindex.h>
#include <init.h>
#include <ui_interface.h>
#include <util.h>
#include <validation.h>
#include <boost/thread.hpp>
constexpr char DB_BEST_BLOCK = 'B';
constexpr char DB_TXINDEX = 't';
constexpr char DB_TXINDEX_BLOCK = 'T';
std::unique_ptr<TxIndex> g_txindex;
struct CDiskTxPos : public CDiskBlockPos
{
unsigned int nTxOffset; // after header
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITEAS(CDiskBlockPos, *this);
READWRITE(VARINT(nTxOffset));
}
CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
}
CDiskTxPos() {
SetNull();
}
void SetNull() {
CDiskBlockPos::SetNull();
nTxOffset = 0;
}
};
/**
* Access to the txindex database (indexes/txindex/)
*
* The database stores a block locator of the chain the database is synced to
* so that the TxIndex can efficiently determine the point it last stopped at.
* A locator is used instead of a simple hash of the chain tip because blocks
* and block index entries may not be flushed to disk until after this database
* is updated.
*/
class TxIndex::DB : public BaseIndex::DB
{
public:
explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
/// Read the disk location of the transaction data with the given hash. Returns false if the
/// transaction hash is not indexed.
bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
/// Write a batch of transaction positions to the DB.
bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
/// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
/// been upgraded yet to the new database.
bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
};
TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
{}
bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
{
return Read(std::make_pair(DB_TXINDEX, txid), pos);
}
bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
{
CDBBatch batch(*this);
for (const auto& tuple : v_pos) {
batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
}
return WriteBatch(batch);
}
/*
* Safely persist a transfer of data from the old txindex database to the new one, and compact the
* range of keys updated. This is used internally by MigrateData.
*/
static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb,
CDBBatch& batch_newdb, CDBBatch& batch_olddb,
const std::pair<unsigned char, uint256>& begin_key,
const std::pair<unsigned char, uint256>& end_key)
{
// Sync new DB changes to disk before deleting from old DB.
newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
olddb.WriteBatch(batch_olddb);
olddb.CompactRange(begin_key, end_key);
batch_newdb.Clear();
batch_olddb.Clear();
}
bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
{
// The prior implementation of txindex was always in sync with block index
// and presence was indicated with a boolean DB flag. If the flag is set,
// this means the txindex from a previous version is valid and in sync with
// the chain tip. The first step of the migration is to unset the flag and
// write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
// index entries are copied over in batches to the new database. Finally,
// DB_TXINDEX_BLOCK is erased from the old database and the block hash is
// written to the new database.
//
// Unsetting the boolean flag ensures that if the node is downgraded to a
// previous version, it will not see a corrupted, partially migrated index
// -- it will see that the txindex is disabled. When the node is upgraded
// again, the migration will pick up where it left off and sync to the block
// with hash DB_TXINDEX_BLOCK.
bool f_legacy_flag = false;
block_tree_db.ReadFlag("txindex", f_legacy_flag);
if (f_legacy_flag) {
if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
return error("%s: cannot write block indicator", __func__);
}
if (!block_tree_db.WriteFlag("txindex", false)) {
return error("%s: cannot write block index db flag", __func__);
}
}
CBlockLocator locator;
if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
return true;
}
int64_t count = 0;
LogPrintf("Upgrading txindex database... [0%%]\n");
uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
int report_done = 0;
const size_t batch_size = 1 << 24; // 16 MiB
CDBBatch batch_newdb(*this);
CDBBatch batch_olddb(block_tree_db);
std::pair<unsigned char, uint256> key;
std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
std::pair<unsigned char, uint256> prev_key = begin_key;
bool interrupted = false;
std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
boost::this_thread::interruption_point();
if (ShutdownRequested()) {
interrupted = true;
break;
}
if (!cursor->GetKey(key)) {
return error("%s: cannot get key from valid cursor", __func__);
}
if (key.first != DB_TXINDEX) {
break;
}
// Log progress every 10%.
if (++count % 256 == 0) {
// Since txids are uniformly random and traversed in increasing order, the high 16 bits
// of the hash can be used to estimate the current progress.
const uint256& txid = key.second;
uint32_t high_nibble =
(static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
(static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
if (report_done < percentage_done/10) {
LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
report_done = percentage_done/10;
}
}
CDiskTxPos value;
if (!cursor->GetValue(value)) {
return error("%s: cannot parse txindex record", __func__);
}
batch_newdb.Write(key, value);
batch_olddb.Erase(key);
if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
// NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
// because LevelDB iterators are guaranteed to provide a consistent view of the
// underlying data, like a lightweight snapshot.
WriteTxIndexMigrationBatches(*this, block_tree_db,
batch_newdb, batch_olddb,
prev_key, key);
prev_key = key;
}
}
// If these final DB batches complete the migration, write the best block
// hash marker to the new database and delete from the old one. This signals
// that the former is fully caught up to that point in the blockchain and
// that all txindex entries have been removed from the latter.
if (!interrupted) {
batch_olddb.Erase(DB_TXINDEX_BLOCK);
batch_newdb.Write(DB_BEST_BLOCK, locator);
}
WriteTxIndexMigrationBatches(*this, block_tree_db,
batch_newdb, batch_olddb,
begin_key, key);
if (interrupted) {
LogPrintf("[CANCELLED].\n");
return false;
}
uiInterface.ShowProgress("", 100, false);
LogPrintf("[DONE].\n");
return true;
}
TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
: m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
{}
TxIndex::~TxIndex() {}
bool TxIndex::Init()
{
LOCK(cs_main);
@ -38,7 +255,7 @@ bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
return m_db->WriteTxs(vPos);
}
BaseIndexDB& TxIndex::GetDB() const { return *m_db; }
BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
{

12
src/index/txindex.h
View file

@ -5,7 +5,9 @@
#ifndef BITCOIN_INDEX_TXINDEX_H
#define BITCOIN_INDEX_TXINDEX_H
#include <chain.h>
#include <index/base.h>
#include <txdb.h>
/**
* TxIndex is used to look up transactions included in the blockchain by hash.
@ -14,8 +16,11 @@
*/
class TxIndex final : public BaseIndex
{
protected:
class DB;
private:
const std::unique_ptr<TxIndexDB> m_db;
const std::unique_ptr<DB> m_db;
protected:
/// Override base class init to migrate from old database.
@ -23,7 +28,7 @@ protected:
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
BaseIndexDB& GetDB() const override;
BaseIndex::DB& GetDB() const override;
const char* GetName() const override { return "txindex"; }
@ -31,6 +36,9 @@ public:
/// Constructs the index, which becomes available to be queried.
explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
// Destructor is declared because this class contains a unique_ptr to an incomplete type.
virtual ~TxIndex() override;
/// Look up a transaction by hash.
///
/// @param[in] tx_hash The hash of the transaction to be returned.

176
src/txdb.cpp
View file

@ -21,8 +21,6 @@
static const char DB_COIN = 'C';
static const char DB_COINS = 'c';
static const char DB_BLOCK_FILES = 'f';
static const char DB_TXINDEX = 't';
static const char DB_TXINDEX_BLOCK = 'T';
static const char DB_BLOCK_INDEX = 'b';
static const char DB_BEST_BLOCK = 'B';
@ -414,177 +412,3 @@ bool CCoinsViewDB::Upgrade() {
LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
return !ShutdownRequested();
}
BaseIndexDB::BaseIndexDB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) :
CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate)
{}
TxIndexDB::TxIndexDB(size_t n_cache_size, bool f_memory, bool f_wipe) :
BaseIndexDB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
{}
bool TxIndexDB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
{
return Read(std::make_pair(DB_TXINDEX, txid), pos);
}
bool TxIndexDB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
{
CDBBatch batch(*this);
for (const auto& tuple : v_pos) {
batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
}
return WriteBatch(batch);
}
bool BaseIndexDB::ReadBestBlock(CBlockLocator& locator) const
{
bool success = Read(DB_BEST_BLOCK, locator);
if (!success) {
locator.SetNull();
}
return success;
}
bool BaseIndexDB::WriteBestBlock(const CBlockLocator& locator)
{
return Write(DB_BEST_BLOCK, locator);
}
/*
* Safely persist a transfer of data from the old txindex database to the new one, and compact the
* range of keys updated. This is used internally by MigrateData.
*/
static void WriteTxIndexMigrationBatches(TxIndexDB& newdb, CBlockTreeDB& olddb,
CDBBatch& batch_newdb, CDBBatch& batch_olddb,
const std::pair<unsigned char, uint256>& begin_key,
const std::pair<unsigned char, uint256>& end_key)
{
// Sync new DB changes to disk before deleting from old DB.
newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
olddb.WriteBatch(batch_olddb);
olddb.CompactRange(begin_key, end_key);
batch_newdb.Clear();
batch_olddb.Clear();
}
bool TxIndexDB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
{
// The prior implementation of txindex was always in sync with block index
// and presence was indicated with a boolean DB flag. If the flag is set,
// this means the txindex from a previous version is valid and in sync with
// the chain tip. The first step of the migration is to unset the flag and
// write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
// index entries are copied over in batches to the new database. Finally,
// DB_TXINDEX_BLOCK is erased from the old database and the block hash is
// written to the new database.
//
// Unsetting the boolean flag ensures that if the node is downgraded to a
// previous version, it will not see a corrupted, partially migrated index
// -- it will see that the txindex is disabled. When the node is upgraded
// again, the migration will pick up where it left off and sync to the block
// with hash DB_TXINDEX_BLOCK.
bool f_legacy_flag = false;
block_tree_db.ReadFlag("txindex", f_legacy_flag);
if (f_legacy_flag) {
if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
return error("%s: cannot write block indicator", __func__);
}
if (!block_tree_db.WriteFlag("txindex", false)) {
return error("%s: cannot write block index db flag", __func__);
}
}
CBlockLocator locator;
if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
return true;
}
int64_t count = 0;
LogPrintf("Upgrading txindex database... [0%%]\n");
uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
int report_done = 0;
const size_t batch_size = 1 << 24; // 16 MiB
CDBBatch batch_newdb(*this);
CDBBatch batch_olddb(block_tree_db);
std::pair<unsigned char, uint256> key;
std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
std::pair<unsigned char, uint256> prev_key = begin_key;
bool interrupted = false;
std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
boost::this_thread::interruption_point();
if (ShutdownRequested()) {
interrupted = true;
break;
}
if (!cursor->GetKey(key)) {
return error("%s: cannot get key from valid cursor", __func__);
}
if (key.first != DB_TXINDEX) {
break;
}
// Log progress every 10%.
if (++count % 256 == 0) {
// Since txids are uniformly random and traversed in increasing order, the high 16 bits
// of the hash can be used to estimate the current progress.
const uint256& txid = key.second;
uint32_t high_nibble =
(static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
(static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
if (report_done < percentage_done/10) {
LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
report_done = percentage_done/10;
}
}
CDiskTxPos value;
if (!cursor->GetValue(value)) {
return error("%s: cannot parse txindex record", __func__);
}
batch_newdb.Write(key, value);
batch_olddb.Erase(key);
if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
// NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
// because LevelDB iterators are guaranteed to provide a consistent view of the
// underlying data, like a lightweight snapshot.
WriteTxIndexMigrationBatches(*this, block_tree_db,
batch_newdb, batch_olddb,
prev_key, key);
prev_key = key;
}
}
// If these final DB batches complete the migration, write the best block
// hash marker to the new database and delete from the old one. This signals
// that the former is fully caught up to that point in the blockchain and
// that all txindex entries have been removed from the latter.
if (!interrupted) {
batch_olddb.Erase(DB_TXINDEX_BLOCK);
batch_newdb.Write(DB_BEST_BLOCK, locator);
}
WriteTxIndexMigrationBatches(*this, block_tree_db,
batch_newdb, batch_olddb,
begin_key, key);
if (interrupted) {
LogPrintf("[CANCELLED].\n");
return false;
}
uiInterface.ShowProgress("", 100, false);
LogPrintf("[DONE].\n");
return true;
}

64
src/txdb.h
View file

@ -40,31 +40,6 @@ static const int64_t nMaxTxIndexCache = 1024;
//! Max memory allocated to coin DB specific cache (MiB)
static const int64_t nMaxCoinsDBCache = 8;
struct CDiskTxPos : public CDiskBlockPos
{
unsigned int nTxOffset; // after header
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITEAS(CDiskBlockPos, *this);
READWRITE(VARINT(nTxOffset));
}
CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
}
CDiskTxPos() {
SetNull();
}
void SetNull() {
CDiskBlockPos::SetNull();
nTxOffset = 0;
}
};
/** CCoinsView backed by the coin database (chainstate/) */
class CCoinsViewDB final : public CCoinsView
{
@ -123,43 +98,4 @@ public:
bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex);
};
class BaseIndexDB : public CDBWrapper
{
public:
BaseIndexDB(const fs::path& path, size_t n_cache_size,
bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
/// Read block locator of the chain that the index is in sync with.
bool ReadBestBlock(CBlockLocator& locator) const;
/// Write block locator of the chain that the index is in sync with.
bool WriteBestBlock(const CBlockLocator& locator);
};
/**
* Access to the txindex database (indexes/txindex/)
*
* The database stores a block locator of the chain the database is synced to
* so that the TxIndex can efficiently determine the point it last stopped at.
* A locator is used instead of a simple hash of the chain tip because blocks
* and block index entries may not be flushed to disk until after this database
* is updated.
*/
class TxIndexDB : public BaseIndexDB
{
public:
explicit TxIndexDB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
/// Read the disk location of the transaction data with the given hash. Returns false if the
/// transaction hash is not indexed.
bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
/// Write a batch of transaction positions to the DB.
bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
/// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
/// been upgraded yet to the new database.
bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
};
#endif // BITCOIN_TXDB_H