From a0221c3bf7ef47c8cd84b6c37094fdf8c0d9d815 Mon Sep 17 00:00:00 2001 From: lbrynaut Date: Mon, 26 Mar 2018 16:34:50 -0500 Subject: [PATCH] Add a claim index for fast reference by id --- src/claimtrie.cpp | 93 ++++++++++++++++++++++++++++++- src/claimtrie.h | 125 +++++++++++++++++++++++++----------------- src/rpc/claimtrie.cpp | 37 ++++++------- 3 files changed, 184 insertions(+), 71 deletions(-) diff --git a/src/claimtrie.cpp b/src/claimtrie.cpp index 42fc55769..92529ce69 100644 --- a/src/claimtrie.cpp +++ b/src/claimtrie.cpp @@ -591,6 +591,48 @@ bool CClaimTrie::recursiveCheckConsistency(const CClaimTrieNode* node) const return calculatedHash == node->hash; } +void CClaimTrie::addToClaimIndex(const std::string& name, const CClaimValue& claim) +{ + // sanity checks only + claimIndexType::const_iterator itClaim = claimIndex.find(claim.claimId); + if (itClaim != claimIndex.end() && itClaim->second.claim != claim) + LogPrintf("%s: WARNING: ClaimIndex Conflict on claim id %s\n", __func__, claim.claimId.GetHex()); + + CClaimIndexElement element = { name, claim }; + claimIndex[claim.claimId] = element; + LogPrintf("%s: ClaimIndex[%s] updated %s\n", __func__, claim.claimId.GetHex(), name); +} + +void CClaimTrie::removeFromClaimIndex(const CClaimValue& claim) +{ + claimIndexType::iterator itClaim = claimIndex.find(claim.claimId); + if (itClaim != claimIndex.end()) + { + LogPrintf("%s: ClaimIndex[%s] removing %s\n", __func__, claim.claimId.GetHex(), itClaim->second.name); + claimIndex.erase(itClaim); + } +} + +bool CClaimTrie::getClaimById(const uint160 claimId, std::string& name, CClaimValue& claim) const +{ + claimIndexType::const_iterator itClaim = claimIndex.find(claimId); + if (itClaim != claimIndex.end()) + { + name = itClaim->second.name; + claim = itClaim->second.claim; + return true; + } + + CClaimIndexElement element; + if (db.Read(std::make_pair(CLAIM_BY_ID, claimId), element)) + { + name = element.name; + claim = element.claim; + return true; + } + return false; +} + bool CClaimTrie::getQueueRow(int nHeight, claimQueueRowType& row) const { claimQueueType::const_iterator itQueueRow = dirtyQueueRows.find(nHeight); @@ -927,6 +969,16 @@ bool CClaimTrie::updateTakeoverHeight(const std::string& name, int nTakeoverHeig return true; } +void CClaimTrie::BatchWriteClaimIndex(CDBBatch& batch) const +{ + for(claimIndexType::const_iterator itClaim = claimIndex.begin(); + itClaim != claimIndex.end(); ++itClaim) + { + batch.Write(std::make_pair(CLAIM_BY_ID, itClaim->second.claim.claimId), itClaim->second); + LogPrintf("%s: Writing %s to disk\n", __func__, itClaim->second.claim.claimId.GetHex()); + } +} + void CClaimTrie::BatchWriteNode(CDBBatch& batch, const std::string& name, const CClaimTrieNode* pNode) const { uint32_t num_claims = 0; @@ -1064,6 +1116,8 @@ bool CClaimTrie::WriteToDisk() dirtySupportQueueNameRows.clear(); BatchWriteSupportExpirationQueueRows(batch); dirtySupportExpirationQueueRows.clear(); + BatchWriteClaimIndex(batch); + claimIndex.clear(); batch.Write(HASH_BLOCK, hashBlock); batch.Write(CURRENT_HEIGHT, nCurrentHeight); return db.WriteBatch(batch); @@ -1085,6 +1139,11 @@ bool CClaimTrie::InsertFromDisk(const std::string& name, CClaimTrieNode* node) current = itchild->second; } current->children[name[name.size()-1]] = node; + + for(size_t i = 0; i < current->claims.size(); ++i) + { + addToClaimIndex(name.substr(0, name.size()-1), current->claims[i]); + } return true; } @@ -1095,6 +1154,32 @@ bool CClaimTrie::ReadFromDisk(bool check) if (!db.Read(CURRENT_HEIGHT, nCurrentHeight)) LogPrintf("%s: Couldn't read the current height\n", __func__); boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + + pcursor->SeekToFirst(); + while (pcursor->Valid()) + { + std::pair key; + if (pcursor->GetKey(key) && (key.first == CLAIM_BY_ID)) + { + CClaimIndexElement element; + if (pcursor->GetValue(element)) + { + claimIndex[element.claim.claimId] = element; + LogPrintf("%s: ClaimIndex Read %s from disk\n", __func__, element.claim.claimId.GetHex()); + } + else + { + return error("%s(): error reading claim index entry from disk", __func__); + } + } + pcursor->Next(); + } + + if (claimIndex.empty()) + LogPrintf("Building ClaimIndex from persistent ClaimTrie...\n"); + else + LogPrintf("Restored ClaimIndex with %lu elements...\n", claimIndex.size()); + pcursor->SeekToFirst(); while (pcursor->Valid()) @@ -1120,6 +1205,7 @@ bool CClaimTrie::ReadFromDisk(bool check) } pcursor->Next(); } + if (check) { LogPrintf("Checking Claim trie consistency..."); @@ -1318,6 +1404,7 @@ bool CClaimTrieCache::insertClaimIntoTrie(const std::string& name, CClaimValue c { fChanged = true; currentNode->insertClaim(claim); + base->addToClaimIndex(name, claim); } else { @@ -1392,7 +1479,8 @@ bool CClaimTrieCache::removeClaimFromTrie(const std::string& name, const COutPoi CClaimValue currentTop = currentNode->claims.front(); success = currentNode->removeClaim(outPoint, claim); - + base->removeFromClaimIndex(claim); + if (!currentNode->claims.empty()) { supportMapEntryType node; @@ -1589,6 +1677,7 @@ bool CClaimTrieCache::addClaimToQueues(const std::string& name, CClaimValue& cla itQueueNameRow->second.push_back(outPointHeightType(claim.outPoint, claim.nValidAtHeight)); nameOutPointType expireEntry(name, claim.outPoint); addToExpirationQueue(claim.nHeight + base->nExpirationTime, expireEntry); + base->addToClaimIndex(name, claim); return true; } @@ -1627,6 +1716,8 @@ bool CClaimTrieCache::removeClaimFromQueue(const std::string& name, const COutPo std::swap(claim, itQueue->second); itQueueNameRow->second.erase(itQueueName); itQueueRow->second.erase(itQueue); + + base->removeFromClaimIndex(claim); return true; } } diff --git a/src/claimtrie.h b/src/claimtrie.h index f41f5e769..6b11d81d9 100644 --- a/src/claimtrie.h +++ b/src/claimtrie.h @@ -10,11 +10,13 @@ #include #include +#include // leveldb keys #define HASH_BLOCK 'h' #define CURRENT_HEIGHT 't' #define TRIE_NODE 'n' +#define CLAIM_BY_ID 'i' #define CLAIM_QUEUE_ROW 'r' #define CLAIM_QUEUE_NAME_ROW 'm' #define EXP_QUEUE_ROW 'e' @@ -54,7 +56,7 @@ public: READWRITE(nHeight); READWRITE(nValidAtHeight); } - + bool operator<(const CClaimValue& other) const { if (nEffectiveAmount < other.nEffectiveAmount) @@ -71,12 +73,12 @@ public: } return false; } - + bool operator==(const CClaimValue& other) const { return outPoint == other.outPoint && claimId == other.claimId && nAmount == other.nAmount && nHeight == other.nHeight && nValidAtHeight == other.nValidAtHeight; } - + bool operator!=(const CClaimValue& other) const { return !(*this == other); @@ -91,7 +93,7 @@ public: CAmount nAmount; int nHeight; int nValidAtHeight; - + CSupportValue() {}; CSupportValue(COutPoint outPoint, uint160 supportedClaimId, CAmount nAmount, int nHeight, int nValidAtHeight) @@ -99,7 +101,7 @@ public: , nAmount(nAmount), nHeight(nHeight) , nValidAtHeight(nValidAtHeight) {} - + ADD_SERIALIZE_METHODS; template @@ -147,7 +149,7 @@ public: bool empty() const {return children.empty() && claims.empty();} bool haveClaim(const COutPoint& outPoint) const; void reorderClaims(supportMapEntryType& supports); - + ADD_SERIALIZE_METHODS; template @@ -156,7 +158,7 @@ public: READWRITE(claims); READWRITE(nHeightOfLastTakeover); } - + bool operator==(const CClaimTrieNode& other) const { return hash == other.hash && claims == other.claims; @@ -241,6 +243,21 @@ struct nameOutPointType } }; +class CClaimIndexElement +{ + public: + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(name); + READWRITE(claim); + } + + std::string name; + CClaimValue claim; +}; + typedef std::pair claimQueueEntryType; typedef std::pair supportQueueEntryType; @@ -265,6 +282,8 @@ typedef std::map nodeCacheType; typedef std::map hashMapType; +typedef std::map claimIndexType; + struct claimsForNameType { std::vector claims; @@ -286,32 +305,36 @@ public: , nProportionalDelayFactor(nProportionalDelayFactor) , root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) {} - + uint256 getMerkleHash(); - + bool empty() const; void clear(); - + bool checkConsistency() const; - + bool WriteToDisk(); bool ReadFromDisk(bool check = false); - + std::vector flattenTrie() const; bool getInfoForName(const std::string& name, CClaimValue& claim) const; bool getLastTakeoverForName(const std::string& name, int& lastTakeoverHeight) const; claimsForNameType getClaimsForName(const std::string& name) const; CAmount getEffectiveAmountForClaim(const std::string& name, uint160 claimId) const; - + bool queueEmpty() const; bool supportEmpty() const; bool supportQueueEmpty() const; bool expirationQueueEmpty() const; bool supportExpirationQueueEmpty() const; - + void setExpirationTime(int t); - + + void addToClaimIndex(const std::string& name, const CClaimValue& claim); + void removeFromClaimIndex(const CClaimValue& claim); + + bool getClaimById(const uint160 claimId, std::string& name, CClaimValue& claim) const; bool getQueueRow(int nHeight, claimQueueRowType& row) const; bool getQueueNameRow(const std::string& name, queueNameRowType& row) const; bool getExpirationQueueRow(int nHeight, expirationQueueRowType& row) const; @@ -319,21 +342,22 @@ public: bool getSupportQueueRow(int nHeight, supportQueueRowType& row) const; bool getSupportQueueNameRow(const std::string& name, queueNameRowType& row) const; bool getSupportExpirationQueueRow(int nHeight, expirationQueueRowType& row) const; - + + bool haveClaim(const std::string& name, const COutPoint& outPoint) const; bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const; - + bool haveSupport(const std::string& name, const COutPoint& outPoint) const; bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const; - + unsigned int getTotalNamesInTrie() const; unsigned int getTotalClaimsInTrie() const; CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const; - + friend class CClaimTrieCache; - + CDBWrapper db; int nCurrentHeight; int nExpirationTime; @@ -342,7 +366,7 @@ private: void clear(CClaimTrieNode* current); const CClaimTrieNode* getNodeForName(const std::string& name) const; - + bool update(nodeCacheType& cache, hashMapType& hashes, std::map& takeoverHeights, const uint256& hashBlock, claimQueueType& queueCache, @@ -356,11 +380,11 @@ private: bool updateHash(const std::string& name, uint256& hash); bool updateTakeoverHeight(const std::string& name, int nTakeoverHeight); bool recursiveNullify(CClaimTrieNode* node, std::string& name); - + bool recursiveCheckConsistency(const CClaimTrieNode* node) const; - + bool InsertFromDisk(const std::string& name, CClaimTrieNode* node); - + unsigned int getTotalNamesRecursive(const CClaimTrieNode* current) const; unsigned int getTotalClaimsRecursive(const CClaimTrieNode* current) const; CAmount getTotalValueOfClaimsRecursive(const CClaimTrieNode* current, @@ -368,7 +392,7 @@ private: bool recursiveFlattenTrie(const std::string& name, const CClaimTrieNode* current, std::vector& nodes) const; - + void markNodeDirty(const std::string& name, CClaimTrieNode* node); void updateQueueRow(int nHeight, claimQueueRowType& row); void updateQueueNameRow(const std::string& name, @@ -379,10 +403,11 @@ private: void updateSupportNameQueue(const std::string& name, queueNameRowType& row); void updateSupportExpirationQueue(int nHeight, expirationQueueRowType& row); - + void BatchWriteNode(CDBBatch& batch, const std::string& name, const CClaimTrieNode* pNode) const; void BatchEraseNode(CDBBatch& batch, const std::string& nome) const; + void BatchWriteClaimIndex(CDBBatch& batch) const; void BatchWriteQueueRows(CDBBatch& batch); void BatchWriteQueueNameRows(CDBBatch& batch); void BatchWriteExpirationQueueRows(CDBBatch& batch); @@ -391,20 +416,22 @@ private: void BatchWriteSupportQueueNameRows(CDBBatch& batch); void BatchWriteSupportExpirationQueueRows(CDBBatch& batch); template bool keyTypeEmpty(char key, K& dummy) const; - + CClaimTrieNode root; uint256 hashBlock; - + claimQueueType dirtyQueueRows; queueNameType dirtyQueueNameRows; expirationQueueType dirtyExpirationQueueRows; - + supportQueueType dirtySupportQueueRows; queueNameType dirtySupportQueueNameRows; expirationQueueType dirtySupportExpirationQueueRows; - + nodeCacheType dirtyNodes; supportMapType dirtySupportNodes; + + mutable claimIndexType claimIndex; }; class CClaimTrieProofNode @@ -441,13 +468,13 @@ public: assert(base); nCurrentHeight = base->nCurrentHeight; } - + uint256 getMerkleHash() const; - + bool empty() const; bool flush(); bool dirty() const { return !dirtyHashes.empty(); } - + bool addClaim(const std::string& name, const COutPoint& outPoint, uint160 claimId, CAmount nAmount, int nHeight) const; bool undoAddClaim(const std::string& name, const COutPoint& outPoint, @@ -457,7 +484,7 @@ public: bool undoSpendClaim(const std::string& name, const COutPoint& outPoint, uint160 claimId, CAmount nAmount, int nHeight, int nValidAtHeight) const; - + bool addSupport(const std::string& name, const COutPoint& outPoint, CAmount nAmount, uint160 supportedClaimId, int nHeight) const; @@ -468,7 +495,7 @@ public: bool undoSpendSupport(const std::string& name, const COutPoint& outPoint, uint160 supportedClaimId, CAmount nAmount, int nHeight, int nValidAtHeight) const; - + uint256 getBestBlock(); void setBestBlock(const uint256& hashBlock); @@ -482,9 +509,9 @@ public: insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo, std::vector >& takeoverHeightUndo) const; - + ~CClaimTrieCache() { clear(); } - + bool insertClaimIntoTrie(const std::string& name, CClaimValue claim, bool fCheckTakeover = false) const; bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, @@ -513,48 +540,48 @@ protected: mutable std::map cacheTakeoverHeights; mutable int nCurrentHeight; // Height of the block that is being worked on, which is // one greater than the height of the chain's tip - + uint256 hashBlock; - + uint256 computeHash() const; - + bool reorderTrieNode(const std::string& name, bool fCheckTakeover) const; bool recursiveComputeMerkleHash(CClaimTrieNode* tnCurrent, std::string sPos) const; bool recursivePruneName(CClaimTrieNode* tnCurrent, unsigned int nPos, std::string sName, bool* pfNullified = NULL) const; - + bool clear() const; - + bool removeClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover) const; - + bool addClaimToQueues(const std::string& name, CClaimValue& claim) const; bool removeClaimFromQueue(const std::string& name, const COutPoint& outPoint, CClaimValue& claim) const; void addToExpirationQueue(int nExpirationHeight, nameOutPointType& entry) const; void removeFromExpirationQueue(const std::string& name, const COutPoint& outPoint, int nHeight) const; - + claimQueueType::iterator getQueueCacheRow(int nHeight, bool createIfNotExists) const; queueNameType::iterator getQueueCacheNameRow(const std::string& name, bool createIfNotExists) const; expirationQueueType::iterator getExpirationQueueCacheRow(int nHeight, bool createIfNotExists) const; - + bool removeSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover) const; bool removeSupportFromMap(const std::string& name, const COutPoint& outPoint, CSupportValue& support, bool fCheckTakeover) const; - + bool insertSupportIntoMap(const std::string& name, CSupportValue support, bool fCheckTakeover) const; - + supportQueueType::iterator getSupportQueueCacheRow(int nHeight, bool createIfNotExists) const; queueNameType::iterator getSupportQueueCacheNameRow(const std::string& name, @@ -571,12 +598,12 @@ protected: void removeSupportFromExpirationQueue(const std::string& name, const COutPoint& outPoint, int nHeight) const; - + bool getSupportsForName(const std::string& name, supportMapEntryType& node) const; bool getLastTakeoverForName(const std::string& name, int& lastTakeoverHeight) const; - + int getDelayForName(const std::string& name) const; uint256 getLeafHashForProof(const std::string& currentPosition, unsigned char nodeChar, diff --git a/src/rpc/claimtrie.cpp b/src/rpc/claimtrie.cpp index cafcf9ef7..e6f0fc5eb 100644 --- a/src/rpc/claimtrie.cpp +++ b/src/rpc/claimtrie.cpp @@ -374,27 +374,22 @@ UniValue getclaimbyid(const UniValue& params, bool fHelp) uint160 claimId; claimId.SetHex(params[0].get_str()); UniValue claim(UniValue::VOBJ); - std::vector nodes = pclaimTrie->flattenTrie(); - for (std::vector::iterator it = nodes.begin(); it != nodes.end(); ++it) { - if (!it->second.claims.empty()) { - for (std::vector::iterator itClaims = it->second.claims.begin(); - itClaims != it->second.claims.end(); ++itClaims) { - if (claimId == itClaims->claimId) { - std::string sValue; - getValueForClaim(itClaims->outPoint, sValue); - claim.push_back(Pair("name", it->first)); - claim.push_back(Pair("value", sValue)); - claim.push_back(Pair("claimId", itClaims->claimId.GetHex())); - claim.push_back(Pair("txid", itClaims->outPoint.hash.GetHex())); - claim.push_back(Pair("n", (int) itClaims->outPoint.n)); - claim.push_back(Pair("amount", itClaims->nAmount)); - claim.push_back(Pair("effective amount", - pclaimTrie->getEffectiveAmountForClaim(it->first, itClaims->claimId))); - claim.push_back(Pair("height", itClaims->nHeight)); - return claim; - } - } - } + std::string name; + CClaimValue claimValue; + pclaimTrie->getClaimById(claimId, name, claimValue); + if (claimValue.claimId == claimId) + { + std::string sValue; + getValueForClaim(claimValue.outPoint, sValue); + claim.push_back(Pair("name", name)); + claim.push_back(Pair("value", sValue)); + claim.push_back(Pair("claimId", claimValue.claimId.GetHex())); + claim.push_back(Pair("txid", claimValue.outPoint.hash.GetHex())); + claim.push_back(Pair("n", (int) claimValue.outPoint.n)); + claim.push_back(Pair("amount", claimValue.nAmount)); + claim.push_back(Pair("effective amount", + pclaimTrie->getEffectiveAmountForClaim(name, claimValue.claimId))); + claim.push_back(Pair("height", claimValue.nHeight)); } return claim; }