LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

changed so each LDB key was in its own DB

Browse source
This commit is contained in:
Brannon King 2019-08-19 08:29:56 -06:00
parent 83319b7f31
commit 558d834f4b
7 changed files with 214 additions and 201 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
src/claimtrie.cpp
View file

@ -127,22 +127,45 @@ void CClaimTrieData::reorderClaims(const supportEntryType& supports)
CClaimTrie::CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor)
{
nProportionalDelayFactor = proportionalDelayFactor;
db.reset(new CDBWrapper(GetDataDir() / "claimtrie", 200 * 1024 * 1024, fMemory, fWipe, false));
if (fWipe) {
fs::remove_all(GetDataDir() / "claimtrie"); // old folder
}
db_TRIE_NODE_BY_HASH.reset(new CDBWrapper(GetDataDir() / "claimtrie_TRIE_NODE_BY_HASH", 240 * 1024 * 1024, fMemory, fWipe, false));
db_TRIE_NODE_BY_NAME.reset(new CDBWrapper(GetDataDir() / "claimtrie_TRIE_NODE_BY_NAME", 120 * 1024 * 1024, fMemory, fWipe, false));
db_CLAIM_BY_ID.reset(new CDBWrapper(GetDataDir() / "claimtrie_CLAIM_BY_ID", 20 * 1024 * 1024, fMemory, fWipe, false));
db_CLAIM_EXP_QUEUE_ROW.reset(new CDBWrapper(GetDataDir() / "claimtrie_CLAIM_EXP_QUEUE_ROW", 16 * 1024 * 1024, fMemory, fWipe, false));
db_CLAIM_QUEUE_ROW.reset(new CDBWrapper(GetDataDir() / "claimtrie_CLAIM_QUEUE_ROW", 16 * 1024 * 1024, fMemory, fWipe, false));
db_CLAIM_QUEUE_NAME_ROW.reset(new CDBWrapper(GetDataDir() / "claimtrie_CLAIM_QUEUE_NAME_ROW", 32 * 1024 * 1024, fMemory, fWipe, false));
db_SUPPORT_EXP_QUEUE_ROW.reset(new CDBWrapper(GetDataDir() / "claimtrie_SUPPORT_EXP_QUEUE_ROW", 12 * 1024 * 1024, fMemory, fWipe, false));
db_SUPPORT_QUEUE_ROW.reset(new CDBWrapper(GetDataDir() / "claimtrie_SUPPORT_QUEUE_ROW", 12 * 1024 * 1024, fMemory, fWipe, false));
db_SUPPORT_QUEUE_NAME_ROW.reset(new CDBWrapper(GetDataDir() / "claimtrie_SUPPORT_QUEUE_NAME_ROW", 24 * 1024 * 1024, fMemory, fWipe, false));
db_SUPPORT.reset(new CDBWrapper(GetDataDir() / "claimtrie_SUPPORT", 24 * 1024 * 1024, fMemory, fWipe, false));
}
bool CClaimTrie::SyncToDisk()
{
return db && db->Sync();
auto success = true;
success &= db_TRIE_NODE_BY_HASH->Sync();
success &= db_TRIE_NODE_BY_NAME->Sync();
success &= db_CLAIM_BY_ID->Sync();
success &= db_CLAIM_EXP_QUEUE_ROW->Sync();
success &= db_CLAIM_QUEUE_ROW->Sync();
success &= db_CLAIM_QUEUE_NAME_ROW->Sync();
success &= db_SUPPORT_EXP_QUEUE_ROW->Sync();
success &= db_SUPPORT_QUEUE_ROW->Sync();
success &= db_SUPPORT_QUEUE_NAME_ROW->Sync();
success &= db_SUPPORT->Sync();
return success;
}
template <typename Key, typename Container>
typename Container::value_type* getQueue(CDBWrapper& db, uint8_t dbkey, const Key& key, Container& queue, bool create)
typename Container::value_type* getQueue(CDBWrapper& db, const Key& key, Container& queue, bool create)
{
auto itQueue = queue.find(key);
if (itQueue != queue.end())
return &(*itQueue);
typename Container::mapped_type row;
if (db.Read(std::make_pair(dbkey, key), row) || create) {
if (db.Read(key, row) || create) {
auto ret = queue.insert(std::make_pair(key, row));
assert(ret.second);
return &(*ret.first);
@ -160,13 +183,13 @@ inline constexpr bool supportedType()
template <>
std::pair<const int, std::vector<queueEntryType<CClaimValue>>>* CClaimTrieCacheBase::getQueueCacheRow(int nHeight, bool createIfNotExists)
{
return getQueue(*(base->db), CLAIM_QUEUE_ROW, nHeight, claimQueueCache, createIfNotExists);
return getQueue(*(base->db_CLAIM_QUEUE_ROW), nHeight, claimQueueCache, createIfNotExists);
}
template <>
std::pair<const int, std::vector<queueEntryType<CSupportValue>>>* CClaimTrieCacheBase::getQueueCacheRow(int nHeight, bool createIfNotExists)
{
return getQueue(*(base->db), SUPPORT_QUEUE_ROW, nHeight, supportQueueCache, createIfNotExists);
return getQueue(*(base->db_SUPPORT_QUEUE_ROW), nHeight, supportQueueCache, createIfNotExists);
}
template <typename T>
@ -179,13 +202,13 @@ std::pair<const int, std::vector<queueEntryType<T>>>* CClaimTrieCacheBase::getQu
template <>
typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow<CClaimValue>(const std::string& name, bool createIfNotExists)
{
return getQueue(*(base->db), CLAIM_QUEUE_NAME_ROW, name, claimQueueNameCache, createIfNotExists);
return getQueue(*(base->db_CLAIM_QUEUE_NAME_ROW), name, claimQueueNameCache, createIfNotExists);
}
template <>
typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow<CSupportValue>(const std::string& name, bool createIfNotExists)
{
return getQueue(*(base->db), SUPPORT_QUEUE_NAME_ROW, name, supportQueueNameCache, createIfNotExists);
return getQueue(*(base->db_SUPPORT_QUEUE_NAME_ROW), name, supportQueueNameCache, createIfNotExists);
}
template <typename T>
@ -198,13 +221,13 @@ typename queueNameType::value_type* CClaimTrieCacheBase::getQueueCacheNameRow(co
template <>
typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueueCacheRow<CClaimValue>(int nHeight, bool createIfNotExists)
{
return getQueue(*(base->db), CLAIM_EXP_QUEUE_ROW, nHeight, expirationQueueCache, createIfNotExists);
return getQueue(*(base->db_CLAIM_EXP_QUEUE_ROW), nHeight, expirationQueueCache, createIfNotExists);
}
template <>
typename expirationQueueType::value_type* CClaimTrieCacheBase::getExpirationQueueCacheRow<CSupportValue>(int nHeight, bool createIfNotExists)
{
return getQueue(*(base->db), SUPPORT_EXP_QUEUE_ROW, nHeight, supportExpirationQueueCache, createIfNotExists);
return getQueue(*(base->db_SUPPORT_EXP_QUEUE_ROW), nHeight, supportExpirationQueueCache, createIfNotExists);
}
template <typename T>
@ -239,7 +262,7 @@ supportEntryType CClaimTrieCacheBase::getSupportsForName(const std::string& name
return sit->second;
supportEntryType supports;
if (base->db->Read(std::make_pair(SUPPORT, name), supports)) // don't trust the try/catch in here
if (base->db_SUPPORT->Read(name, supports)) // don't trust the try/catch in here
return supports;
return {};
}
@ -489,7 +512,7 @@ std::vector<std::pair<std::string, CClaimTrieDataNode>> CClaimTrie::nodes(const
}
bool CClaimTrie::contains(const std::string &key) const {
return db->Exists(std::make_pair(TRIE_NODE_BY_NAME, key));
return db_TRIE_NODE_BY_NAME->Exists(key);
}
bool CClaimTrie::empty() const {
@ -498,20 +521,20 @@ bool CClaimTrie::empty() const {
bool CClaimTrie::find(const std::string &key, CClaimTrieDataNode &node) const {
uint256 hash;
if (!db->Read(std::make_pair(TRIE_NODE_BY_NAME, key), hash))
if (!db_TRIE_NODE_BY_NAME->Read(key, hash))
return false;
auto found = find(hash, node);
return found;
}
bool CClaimTrie::find(const uint256 &key, CClaimTrieDataNode &node) const {
return db->Read(std::make_pair(TRIE_NODE_BY_HASH, key), node);
return db_TRIE_NODE_BY_HASH->Read(key, node);
}
bool CClaimTrieCacheBase::getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const
{
CClaimIndexElement element;
if (!base->db->Read(std::make_pair(CLAIM_BY_ID, claimId), element))
if (!base->db_CLAIM_BY_ID->Read(claimId, element))
return false;
if (element.claim.claimId == claimId) {
name = element.name;
@ -522,25 +545,27 @@ bool CClaimTrieCacheBase::getClaimById(const uint160& claimId, std::string& name
}
template <typename K, typename T>
void BatchWrite(CDBBatch& batch, uint8_t dbkey, const K& key, const std::vector<T>& value)
void BatchWrite(CDBBatch& batch, const K& key, const std::vector<T>& value)
{
if (value.empty()) {
batch.Erase(std::make_pair(dbkey, key));
batch.Erase(key);
} else {
batch.Write(std::make_pair(dbkey, key), value);
batch.Write(key, value);
}
}
template <typename Container>
void BatchWriteQueue(CDBBatch& batch, uint8_t dbkey, const Container& queue)
bool BatchWriteQueue(CDBWrapper& db, const Container& queue)
{
CDBBatch batch(db);
for (auto& itQueue : queue)
BatchWrite(batch, dbkey, itQueue.first, itQueue.second);
BatchWrite(batch, itQueue.first, itQueue.second);
return db.WriteBatch(batch);
}
bool CClaimTrieCacheBase::flush()
{
CDBBatch batch(*(base->db));
CDBBatch batch_CLAIM_BY_ID(*(base->db_CLAIM_BY_ID));
for (const auto& claim : claimsToDeleteFromByIdIndex) {
auto it = std::find_if(claimsToAddToByIdIndex.begin(), claimsToAddToByIdIndex.end(),
@ -549,11 +574,13 @@ bool CClaimTrieCacheBase::flush()
}
);
if (it == claimsToAddToByIdIndex.end())
batch.Erase(std::make_pair(CLAIM_BY_ID, claim.claimId));
batch_CLAIM_BY_ID.Erase(claim.claimId);
}
for (const auto& e : claimsToAddToByIdIndex)
batch.Write(std::make_pair(CLAIM_BY_ID, e.claim.claimId), e);
batch_CLAIM_BY_ID.Write(e.claim.claimId, e);
auto success = base->db_CLAIM_BY_ID->WriteBatch(batch_CLAIM_BY_ID);
getMerkleHash();
@ -566,6 +593,8 @@ bool CClaimTrieCacheBase::flush()
forDeletion.insert(node.first);
}
CDBBatch batch_TRIE_NODE_BY_HASH(*(base->db_TRIE_NODE_BY_HASH));
CDBBatch batch_TRIE_NODE_BY_NAME(*(base->db_TRIE_NODE_BY_NAME));
for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it) {
forDeletion.erase(it.key());
if (!dirtyNodes.count(it.key()))
@ -576,32 +605,35 @@ bool CClaimTrieCacheBase::flush()
for (auto &child: it.children()) // ordering here is important
node.children.emplace_back(child.key().substr(it.key().size()), child->hash);
batch.Write(std::make_pair(TRIE_NODE_BY_HASH, it->hash), node);
batch.Write(std::make_pair(TRIE_NODE_BY_NAME, it.key()), it->hash);
batch_TRIE_NODE_BY_HASH.Write(it->hash, node);
batch_TRIE_NODE_BY_NAME.Write(it.key(), it->hash);
}
success &= base->db_TRIE_NODE_BY_HASH->WriteBatch(batch_TRIE_NODE_BY_HASH);
for (auto& name: forDeletion) {
batch.Erase(std::make_pair(TRIE_NODE_BY_NAME, name));
batch_TRIE_NODE_BY_NAME.Erase(name);
}
BatchWriteQueue(batch, SUPPORT, supportCache);
success &= base->db_TRIE_NODE_BY_NAME->WriteBatch(batch_TRIE_NODE_BY_NAME);
BatchWriteQueue(batch, CLAIM_QUEUE_ROW, claimQueueCache);
BatchWriteQueue(batch, CLAIM_QUEUE_NAME_ROW, claimQueueNameCache);
BatchWriteQueue(batch, CLAIM_EXP_QUEUE_ROW, expirationQueueCache);
success &= BatchWriteQueue(*(base->db_SUPPORT), supportCache);
BatchWriteQueue(batch, SUPPORT_QUEUE_ROW, supportQueueCache);
BatchWriteQueue(batch, SUPPORT_QUEUE_NAME_ROW, supportQueueNameCache);
BatchWriteQueue(batch, SUPPORT_EXP_QUEUE_ROW, supportExpirationQueueCache);
success &= BatchWriteQueue(*(base->db_CLAIM_QUEUE_ROW), claimQueueCache);
success &= BatchWriteQueue(*(base->db_CLAIM_QUEUE_NAME_ROW), claimQueueNameCache);
success &= BatchWriteQueue(*(base->db_CLAIM_EXP_QUEUE_ROW), expirationQueueCache);
success &= BatchWriteQueue(*(base->db_SUPPORT_QUEUE_ROW), supportQueueCache);
success &= BatchWriteQueue(*(base->db_SUPPORT_QUEUE_NAME_ROW), supportQueueNameCache);
success &= BatchWriteQueue(*(base->db_SUPPORT_EXP_QUEUE_ROW), supportExpirationQueueCache);
base->nNextHeight = nNextHeight;
if (!nodesToAddOrUpdate.empty() && (LogAcceptCategory(BCLog::CLAIMS) || LogAcceptCategory(BCLog::BENCH))) {
LogPrintf("TrieCache size: %zu nodes on block %d, batch writes %zu bytes.\n",
nodesToAddOrUpdate.height(), nNextHeight, batch.SizeEstimate());
LogPrintf("TrieCache size: %zu nodes on block %d.\n",
nodesToAddOrUpdate.height(), nNextHeight);
}
auto ret = base->db->WriteBatch(batch);
clear();
return ret;
return success;
}
bool CClaimTrieCacheBase::validateTrieConsistency(const CBlockIndex* tip)
@ -623,7 +655,7 @@ bool CClaimTrieCacheBase::ReadFromDisk(const CBlockIndex* tip)
base->nNextHeight = nNextHeight = tip ? tip->nHeight + 1 : 0;
clear();
if (tip && (base->db->Exists(std::make_pair(TRIE_NODE, std::string())) || !base->db->Exists(std::make_pair(TRIE_NODE_BY_HASH, tip->hashClaimTrie)))) {
if (tip && fs::is_directory(GetDataDir() / "claimtrie")) {
LogPrintf("The claim trie database contains deprecated data and will need to be rebuilt");
return false;
}

24
src/claimtrie.h
View file

@ -17,19 +17,6 @@
#include <unordered_map>
#include <unordered_set>
// leveldb keys
#define TRIE_NODE 'n' // deprecated
#define TRIE_NODE_BY_HASH 'h'
#define TRIE_NODE_BY_NAME 'g'
#define CLAIM_BY_ID 'i'
#define CLAIM_QUEUE_ROW 'r'
#define CLAIM_QUEUE_NAME_ROW 'm'
#define CLAIM_EXP_QUEUE_ROW 'e'
#define SUPPORT 's'
#define SUPPORT_QUEUE_ROW 'u'
#define SUPPORT_QUEUE_NAME_ROW 'p'
#define SUPPORT_EXP_QUEUE_ROW 'x'
uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover);
struct CClaimValue
@ -321,7 +308,16 @@ class CClaimTrie
{
int nNextHeight = 0;
int nProportionalDelayFactor = 0;
std::unique_ptr<CDBWrapper> db;
std::unique_ptr<CDBWrapper> db_TRIE_NODE_BY_HASH;
std::unique_ptr<CDBWrapper> db_TRIE_NODE_BY_NAME;
std::unique_ptr<CDBWrapper> db_CLAIM_BY_ID;
std::unique_ptr<CDBWrapper> db_CLAIM_EXP_QUEUE_ROW;
std::unique_ptr<CDBWrapper> db_CLAIM_QUEUE_ROW;
std::unique_ptr<CDBWrapper> db_CLAIM_QUEUE_NAME_ROW;
std::unique_ptr<CDBWrapper> db_SUPPORT_EXP_QUEUE_ROW;
std::unique_ptr<CDBWrapper> db_SUPPORT_QUEUE_ROW;
std::unique_ptr<CDBWrapper> db_SUPPORT_QUEUE_NAME_ROW;
std::unique_ptr<CDBWrapper> db_SUPPORT;
public:
CClaimTrie() = default;

49
src/claimtrieforks.cpp
View file

@ -61,29 +61,31 @@ bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
*/
//look through db for expiration queues, if we haven't already found it in dirty expiration queue
boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
std::pair<uint8_t, int> key;
if (!pcursor->GetKey(key))
boost::scoped_ptr<CDBIterator> pcursor_ceqr(base->db_CLAIM_EXP_QUEUE_ROW->NewIterator());
for (pcursor_ceqr->SeekToFirst(); pcursor_ceqr->Valid(); pcursor_ceqr->Next()) {
int height;
if (!pcursor_ceqr->GetKey(height))
continue;
int height = key.second;
if (key.first == CLAIM_EXP_QUEUE_ROW) {
expirationQueueRowType row;
if (pcursor->GetValue(row)) {
reactivateClaim(row, height, increment);
} else {
return error("%s(): error reading expiration queue rows from disk", __func__);
}
} else if (key.first == SUPPORT_EXP_QUEUE_ROW) {
expirationQueueRowType row;
if (pcursor->GetValue(row)) {
reactivateSupport(row, height, increment);
} else {
return error("%s(): error reading support expiration queue rows from disk", __func__);
}
expirationQueueRowType row;
if (pcursor_ceqr->GetValue(row)) {
reactivateClaim(row, height, increment);
} else {
return error("%s(): error reading expiration queue rows from disk", __func__);
}
}
boost::scoped_ptr<CDBIterator> pcursor_seqr(base->db_SUPPORT_EXP_QUEUE_ROW->NewIterator());
for (pcursor_seqr->SeekToFirst(); pcursor_seqr->Valid(); pcursor_seqr->Next()) {
int height;
if (!pcursor_seqr->GetKey(height))
continue;
expirationQueueRowType row;
if (pcursor_seqr->GetValue(row)) {
reactivateSupport(row, height, increment);
} else {
return error("%s(): error reading support expiration queue rows from disk", __func__);
}
}
return true;
}
@ -161,15 +163,14 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insert
// run the one-time upgrade of all names that need to change
// it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
boost::scoped_ptr<CDBIterator> pcursor(base->db_TRIE_NODE_BY_NAME->NewIterator());
for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
std::pair<uint8_t, std::string> key;
if (!pcursor->GetKey(key) || key.first != TRIE_NODE_BY_NAME)
std::string name;
if (!pcursor->GetKey(name))
continue;
const auto& name = key.second;
const std::string normalized = normalizeClaimName(name, true);
if (normalized == key.second)
if (normalized == name)
continue;
auto supports = getSupportsForName(name);

7
src/dbwrapper.cpp
View file

@ -116,7 +116,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
}
CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
: m_name(fs::basename(path))
: m_name(fs::basename(path)), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION)
{
penv = nullptr;
readoptions.verify_checksums = true;
@ -239,6 +239,11 @@ bool CDBIterator::Valid() const { return piter->Valid(); }
void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
void CDBIterator::Next() { piter->Next(); }
bool CDBWrapper::Sync() {
CDBBatch batch(*this);
return WriteBatch(batch, true);
}
namespace dbwrapper_private {
void HandleError(const leveldb::Status& status)

197
src/dbwrapper.h
View file

@ -16,9 +16,6 @@
#include <leveldb/db.h>
#include <leveldb/write_batch.h>
static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;
class dbwrapper_error : public std::runtime_error
{
public:
@ -43,77 +40,6 @@ const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
};
/** Batch of changes queued to be written to a CDBWrapper */
class CDBBatch
{
friend class CDBWrapper;
private:
const CDBWrapper &parent;
leveldb::WriteBatch batch;
CDataStream ssKey;
CDataStream ssValue;
size_t size_estimate;
public:
/**
* @param[in] _parent CDBWrapper that this batch is to be submitted to
*/
explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { };
void Clear()
{
batch.Clear();
size_estimate = 0;
}
template <typename K, typename V>
void Write(const K& key, const V& value)
{
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE);
ssValue << value;
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
leveldb::Slice slValue(ssValue.data(), ssValue.size());
batch.Put(slKey, slValue);
// LevelDB serializes writes as:
// - byte: header
// - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
// - byte[]: key
// - varint: value length
// - byte[]: value
// The formula below assumes the key and value are both less than 16k.
size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
ssKey.clear();
ssValue.clear();
}
template <typename K>
void Erase(const K& key)
{
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
batch.Delete(slKey);
// LevelDB serializes erases as:
// - byte: header
// - varint: key length
// - byte[]: key
// The formula below assumes the key is less than 16kB.
size_estimate += 2 + (slKey.size() > 127) + slKey.size();
ssKey.clear();
}
size_t SizeEstimate() const { return size_estimate; }
};
class CDBIterator
{
private:
@ -135,10 +61,10 @@ public:
void SeekToFirst();
template<typename K> void Seek(const K& key) {
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
CDataStream ssKey1(SER_DISK, CLIENT_VERSION);
// ssKey1.reserve(parent.ssKey.capacity());
ssKey1 << key;
leveldb::Slice slKey(ssKey1.data(), ssKey1.size());
piter->Seek(slKey);
}
@ -173,6 +99,8 @@ public:
};
class CDBBatch;
class CDBWrapper
{
friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
@ -213,6 +141,8 @@ private:
std::vector<unsigned char> CreateObfuscateKey() const;
public:
mutable CDataStream ssKey, ssValue;
/**
* @param[in] path Location in the filesystem where leveldb data will be stored.
* @param[in] nCacheSize Configures various leveldb cache settings.
@ -230,13 +160,12 @@ public:
template <typename K, typename V>
bool Read(const K& key, V& value) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
std::string strValue;
leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
ssKey.clear();
if (!status.ok()) {
if (status.IsNotFound())
return false;
@ -254,23 +183,17 @@ public:
}
template <typename K, typename V>
bool Write(const K& key, const V& value, bool fSync = false)
{
CDBBatch batch(*this);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}
bool Write(const K& key, const V& value, bool fSync = false);
template <typename K>
bool Exists(const K& key) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
std::string strValue;
leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
ssKey.clear();
if (!status.ok()) {
if (status.IsNotFound())
return false;
@ -281,12 +204,7 @@ public:
}
template <typename K>
bool Erase(const K& key, bool fSync = false)
{
CDBBatch batch(*this);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
bool Erase(const K& key, bool fSync = false);
bool WriteBatch(CDBBatch& batch, bool fSync = false);
@ -299,11 +217,7 @@ public:
return true;
}
bool Sync()
{
CDBBatch batch(*this);
return WriteBatch(batch, true);
}
bool Sync();
CDBIterator *NewIterator()
{
@ -319,8 +233,6 @@ public:
size_t EstimateSize(const K& key_begin, const K& key_end) const
{
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey1 << key_begin;
ssKey2 << key_end;
leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
@ -338,8 +250,8 @@ public:
void CompactRange(const K& key_begin, const K& key_end) const
{
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey1.reserve(ssKey.capacity());
ssKey2.reserve(ssKey.capacity());
ssKey1 << key_begin;
ssKey2 << key_end;
leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
@ -349,4 +261,83 @@ public:
};
/** Batch of changes queued to be written to a CDBWrapper */
class CDBBatch
{
friend class CDBWrapper;
const CDBWrapper &parent;
leveldb::WriteBatch batch;
size_t size_estimate;
public:
/**
* @param[in] _parent CDBWrapper that this batch is to be submitted to
*/
explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), size_estimate(0) { };
void Clear()
{
batch.Clear();
size_estimate = 0;
}
template <typename K, typename V>
void Write(const K& key, const V& value)
{
assert(parent.ssKey.empty());
parent.ssKey << key;
leveldb::Slice slKey(parent.ssKey.data(), parent.ssKey.size());
assert(parent.ssValue.empty());
parent.ssValue << value;
parent.ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
leveldb::Slice slValue(parent.ssValue.data(), parent.ssValue.size());
batch.Put(slKey, slValue);
// LevelDB serializes writes as:
// - byte: header
// - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
// - byte[]: key
// - varint: value length
// - byte[]: value
// The formula below assumes the key and value are both less than 16k.
size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
parent.ssKey.clear();
parent.ssValue.clear();
}
template <typename K>
void Erase(const K& key)
{
parent.ssKey << key;
leveldb::Slice slKey(parent.ssKey.data(), parent.ssKey.size());
batch.Delete(slKey);
// LevelDB serializes erases as:
// - byte: header
// - varint: key length
// - byte[]: key
// The formula below assumes the key is less than 16kB.
size_estimate += 2 + (slKey.size() > 127) + slKey.size();
parent.ssKey.clear();
}
size_t SizeEstimate() const { return size_estimate; }
};
template<typename K>
bool CDBWrapper::Erase(const K &key, bool fSync) {
CDBBatch batch(*this);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
template<typename K, typename V>
bool CDBWrapper::Write(const K &key, const V &value, bool fSync) {
CDBBatch batch(*this);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}
#endif // BITCOIN_DBWRAPPER_H

5
src/streams.h
View file

@ -227,6 +227,11 @@ public:
return (std::string(begin(), end()));
}
std::size_t capacity() const
{
return vch.capacity();
}
//
// Vector subset

25
src/test/claimtriebranching_tests.cpp
View file

@ -319,30 +319,13 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork
DecrementBlocks(chainActive.Height() - mark);
}
template <typename K>
bool keyTypeEmpty(uint8_t keyType)
{
boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
pcursor->SeekToFirst();
while (pcursor->Valid()) {
std::pair<uint8_t, K> key;
if (pcursor->GetKey(key)) {
if (key.first == keyType)
return false;
}
pcursor->Next();
}
return true;
}
bool queueEmpty()
{
for (const auto& claimQueue: claimQueueCache) {
if (!claimQueue.second.empty())
return false;
}
return keyTypeEmpty<int>(CLAIM_QUEUE_ROW);
return base->db_CLAIM_QUEUE_ROW->IsEmpty();
}
bool expirationQueueEmpty()
@ -351,7 +334,7 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork
if (!expirationQueue.second.empty())
return false;
}
return keyTypeEmpty<int>(CLAIM_EXP_QUEUE_ROW);
return base->db_CLAIM_EXP_QUEUE_ROW->IsEmpty();
}
bool supportEmpty()
@ -360,7 +343,7 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork
if (!entry.second.empty())
return false;
}
return supportCache.empty() && keyTypeEmpty<std::string>(SUPPORT);
return supportCache.empty() && base->db_SUPPORT->IsEmpty();
}
bool supportQueueEmpty()
@ -369,7 +352,7 @@ struct ClaimTrieChainFixture: public CClaimTrieCacheExpirationFork
if (!support.second.empty())
return false;
}
return keyTypeEmpty<int>(SUPPORT_QUEUE_ROW);
return base->db_SUPPORT_QUEUE_ROW->IsEmpty();
}
int proportionalDelayFactor()