From 26125f1351f78854128a43f8e794d4dcad1c2931 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Wed, 8 Jan 2020 15:19:36 -0700 Subject: [PATCH] fixed missing items in block_info, incorrect column names --- src/arith_uint256.h | 4 +-- src/claimtrie/blob.cpp | 12 ++++++-- src/claimtrie/blob.h | 2 +- src/claimtrie/sqlite.h | 14 ++++++++- src/claimtrie/trie.cpp | 56 ++++++++++++++++++----------------- src/claimtrie/uints.cpp | 5 ++++ src/claimtrie/uints.h | 1 + src/init.cpp | 2 +- src/test/claimtriefixture.cpp | 2 +- src/test/uint256_tests.cpp | 16 ++++++++++ src/txdb.cpp | 26 ++++++++++------ src/validation.cpp | 6 ++++ src/validation.h | 2 +- 13 files changed, 102 insertions(+), 46 deletions(-) diff --git a/src/arith_uint256.h b/src/arith_uint256.h index bd0360087..1d2d03fb2 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -57,8 +57,8 @@ public: { static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - pn[0] = (unsigned int)b; - pn[1] = (unsigned int)(b >> 32); + pn[0] = uint32_t(b); + pn[1] = uint32_t(b >> 32U); for (int i = 2; i < WIDTH; i++) pn[i] = 0; } diff --git a/src/claimtrie/blob.cpp b/src/claimtrie/blob.cpp index b492f7887..cd77827a3 100644 --- a/src/claimtrie/blob.cpp +++ b/src/claimtrie/blob.cpp @@ -72,12 +72,18 @@ void CBaseBlob::SetNull() } template -std::string CBaseBlob::GetHex() const +std::string CBaseBlob::GetHex(bool reverse) const { std::stringstream ss; ss << std::hex; - for (auto it = data.rbegin(); it != data.rend(); ++it) - ss << std::setw(2) << std::setfill('0') << uint32_t(*it); + if (reverse) { + for (auto it = data.rbegin(); it != data.rend(); ++it) + ss << std::setw(2) << std::setfill('0') << uint32_t(*it); + } + else { + for (auto it = data.begin(); it != data.end(); ++it) + ss << std::setw(2) << std::setfill('0') << uint32_t(*it); + } return ss.str(); } diff --git a/src/claimtrie/blob.h b/src/claimtrie/blob.h index 7e4ed47e0..20df687d9 100644 --- a/src/claimtrie/blob.h +++ b/src/claimtrie/blob.h @@ -38,7 +38,7 @@ public: bool IsNull() const; void SetNull(); - std::string GetHex() const; + std::string GetHex(bool reverse=true) const; void SetHex(const char* psz); void SetHex(const std::string& str); diff --git a/src/claimtrie/sqlite.h b/src/claimtrie/sqlite.h index 83c1ee5f3..5156480db 100644 --- a/src/claimtrie/sqlite.h +++ b/src/claimtrie/sqlite.h @@ -49,10 +49,22 @@ namespace sqlite inline uint256 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type) { uint256 ret; + +// I think we need this, but I lost my specific test case: +// auto type = sqlite3_column_type(stmt, inx); +// if (type == SQLITE_NULL) +// return ret; +// +// if (type == SQLITE_INTEGER) { +// auto integer = sqlite3_column_int64(stmt, inx); +// return uint256(integer); +// } +// assert(type == SQLITE_BLOB); + auto ptr = sqlite3_column_blob(stmt, inx); if (!ptr) return ret; int bytes = sqlite3_column_bytes(stmt, inx); - assert(bytes > 0 && bytes <= int(ret.size())); + assert(bytes <= ret.size()); std::memcpy(ret.begin(), ptr, bytes); return ret; } diff --git a/src/claimtrie/trie.cpp b/src/claimtrie/trie.cpp index 4695484fd..a55707919 100644 --- a/src/claimtrie/trie.cpp +++ b/src/claimtrie/trie.cpp @@ -77,34 +77,19 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, "parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " "hash BLOB)"; - db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)"; - // db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain - db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)"; - - db << "CREATE TABLE IF NOT EXISTS takeover (name BLOB NOT NULL, height INTEGER NOT NULL, " - "claimID BLOB, PRIMARY KEY(name, height DESC));"; - - db << "CREATE INDEX IF NOT EXISTS takeover_height ON takeover (height)"; - db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL PRIMARY KEY, name BLOB NOT NULL, " "nodeName BLOB NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " "txID BLOB NOT NULL, txN INTEGER NOT NULL, originalHeight INTEGER NOT NULL, updateHeight INTEGER NOT NULL, " "validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, " "expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL);"; - db << "CREATE INDEX IF NOT EXISTS claim_activationHeight ON claim (activationHeight)"; - db << "CREATE INDEX IF NOT EXISTS claim_expirationHeight ON claim (expirationHeight)"; - db << "CREATE INDEX IF NOT EXISTS claim_nodeName ON claim (nodeName)"; - db << "CREATE TABLE IF NOT EXISTS support (txID BLOB NOT NULL, txN INTEGER NOT NULL, " "supportedClaimID BLOB NOT NULL, name BLOB NOT NULL, nodeName BLOB NOT NULL, " "blockHeight INTEGER NOT NULL, validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, " "expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL, PRIMARY KEY(txID, txN));"; - db << "CREATE INDEX IF NOT EXISTS support_supportedClaimID ON support (supportedClaimID)"; - db << "CREATE INDEX IF NOT EXISTS support_activationHeight ON support (activationHeight)"; - db << "CREATE INDEX IF NOT EXISTS support_expirationHeight ON support (expirationHeight)"; - db << "CREATE INDEX IF NOT EXISTS support_nodeName ON support (nodeName)"; + db << "CREATE TABLE IF NOT EXISTS takeover (name BLOB NOT NULL, height INTEGER NOT NULL, " + "claimID BLOB, PRIMARY KEY(name, height DESC));"; if (fWipe) { db << "DELETE FROM node"; @@ -113,6 +98,21 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, db << "DELETE FROM takeover"; } + db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)"; + // db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain + db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)"; + + db << "CREATE INDEX IF NOT EXISTS takeover_height ON takeover (height)"; + + db << "CREATE INDEX IF NOT EXISTS claim_activationHeight ON claim (activationHeight)"; + db << "CREATE INDEX IF NOT EXISTS claim_expirationHeight ON claim (expirationHeight)"; + db << "CREATE INDEX IF NOT EXISTS claim_nodeName ON claim (nodeName)"; + + db << "CREATE INDEX IF NOT EXISTS support_supportedClaimID ON support (supportedClaimID)"; + db << "CREATE INDEX IF NOT EXISTS support_activationHeight ON support (activationHeight)"; + db << "CREATE INDEX IF NOT EXISTS support_expirationHeight ON support (expirationHeight)"; + db << "CREATE INDEX IF NOT EXISTS support_nodeName ON support (nodeName)"; + db << "INSERT OR IGNORE INTO node(name, hash) VALUES(x'', ?)" << one; // ensure that we always have our root node } @@ -462,9 +462,7 @@ bool CClaimTrieCacheBase::validateDb(int height, const uint256& rootHash) { base->nNextHeight = nNextHeight = height + 1; - logPrint << "Checking claim trie consistency... " << Clog::flush; if (checkConsistency()) { - logPrint << "consistent" << Clog::endl; if (rootHash != getMerkleHash()) { logPrint << "CClaimTrieCacheBase::" << __func__ << "(): the block's root claim hash doesn't match the persisted claim root hash." << Clog::endl; return false; @@ -475,7 +473,6 @@ bool CClaimTrieCacheBase::validateDb(int height, const uint256& rootHash) return true; } - logPrint << "inconsistent!" << Clog::endl; return false; } @@ -696,13 +693,18 @@ bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, const COutPoint& o bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight) { + { + auto query = db << "SELECT nodeName, activationHeight FROM support " + "WHERE txID = ? AND txN = ? AND expirationHeight >= ?" + << outPoint.hash << outPoint.n << nNextHeight; + auto it = query.begin(); + if (it == query.end()) + return false; + + *it >> nodeName >> validHeight; + } ensureTransacting(); - auto query = db << "SELECT nodeName, activationHeight FROM support WHERE txID = ? AND txN = ? AND expirationHeight >= ?" - << outPoint.hash << outPoint.n << nNextHeight; - auto it = query.begin(); - if (it == query.end()) return false; - *it >> nodeName >> validHeight; db << "DELETE FROM support WHERE txID = ? AND txN = ?" << outPoint.hash << outPoint.n; if (!db.rows_modified()) return false; @@ -941,7 +943,7 @@ std::vector CClaimTrieCacheBase::getActivatedClaims(int height) const std::vector CClaimTrieCacheBase::getClaimsWithActivatedSupports(int height) const { std::vector ret; - auto query = db << "SELECT DISTINCT supportedClaimID FROM support WHERE activationHeight = ?1 AND updateHeight < ?1" << height; + auto query = db << "SELECT DISTINCT supportedClaimID FROM support WHERE activationHeight = ?1 AND blockHeight < ?1" << height; for (auto&& row: query) { ret.emplace_back(); row >> ret.back(); @@ -963,7 +965,7 @@ std::vector CClaimTrieCacheBase::getExpiredClaims(int height) const std::vector CClaimTrieCacheBase::getClaimsWithExpiredSupports(int height) const { std::vector ret; - auto query = db << "SELECT DISTINCT supportedClaimID FROM support WHERE expirationHeight = ?1 AND updateHeight < ?1" << height; + auto query = db << "SELECT DISTINCT supportedClaimID FROM support WHERE expirationHeight = ?1 AND blockHeight < ?1" << height; for (auto&& row: query) { ret.emplace_back(); row >> ret.back(); diff --git a/src/claimtrie/uints.cpp b/src/claimtrie/uints.cpp index 169cc726f..5c5c35029 100644 --- a/src/claimtrie/uints.cpp +++ b/src/claimtrie/uints.cpp @@ -1,5 +1,6 @@ #include +#include uint160::uint160(const std::vector& vec) : CBaseBlob<160>(vec) { @@ -9,6 +10,10 @@ uint256::uint256(const std::vector& vec) : CBaseBlob<256>(vec) { } +uint256::uint256(int64_t value) : CBaseBlob<256>() { + std::memcpy(this->begin(), &value, sizeof(value)); // TODO: fix the endianness here +} + uint160 uint160S(const char* str) { uint160 s; diff --git a/src/claimtrie/uints.h b/src/claimtrie/uints.h index 5edf1d052..c6de8fd79 100644 --- a/src/claimtrie/uints.h +++ b/src/claimtrie/uints.h @@ -25,6 +25,7 @@ public: uint256() = default; explicit uint256(const std::vector& vec); + explicit uint256(int64_t value); uint256(uint256&&) = default; uint256& operator=(uint256&&) = default; diff --git a/src/init.cpp b/src/init.cpp index 9a878c8da..0d3f71bb3 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1575,7 +1575,7 @@ bool AppInitMain(InitInterfaces& interfaces) auto tip = ::ChainActive().Tip(); if (tip && !::ClaimtrieCache().validateDb(tip->nHeight, tip->hashClaimTrie)) { - strLoadError = _("Error validating the claim trie from disk").translated; + strLoadError = _("Error validating the stored claim trie").translated; break; } diff --git a/src/test/claimtriefixture.cpp b/src/test/claimtriefixture.cpp index 6ea3eaf72..576eadf43 100644 --- a/src/test/claimtriefixture.cpp +++ b/src/test/claimtriefixture.cpp @@ -435,7 +435,7 @@ boost::test_tools::predicate_result ClaimTrieChainFixture::best_claim_effective_ bool ClaimTrieChainFixture::getClaimById(const uint160 &claimId, std::string &name, CClaimValue &value) { - auto query = db << "SELECT nodeName, claimID, txID, txN, amount, validHeight, blockHeight " + auto query = db << "SELECT nodeName, claimID, txID, txN, amount, validHeight, updateHeight " "FROM claim WHERE claimID = ?" << claimId; auto hit = false; for (auto&& row: query) { diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 33a118c2b..24cb77484 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -277,4 +277,20 @@ BOOST_AUTO_TEST_CASE( operator_with_self ) BOOST_CHECK(v == UintToArith256(uint256S("0"))); } +BOOST_AUTO_TEST_CASE( bit64_all_ways ) +{ + std::vector tests = { 0, 1, -1, 42, 1LL << 33U, std::numeric_limits::max(), + std::numeric_limits::lowest() }; + + for (auto test: tests) { + auto a = uint256(test); + auto b = arith_uint256(uint64_t(test)); + auto c = ArithToUint256(b); + auto d = UintToArith256(a); + BOOST_CHECK(a == c); + BOOST_CHECK(b == d); + BOOST_CHECK_EQUAL(a.GetHex(false), c.GetHex(false)); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.cpp b/src/txdb.cpp index 4b01efe9b..46a237e27 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -52,7 +52,7 @@ CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, b bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { auto query = db << "SELECT isCoinbase, blockHeight, amount, script FROM unspent " - "WHERE txID = ? and txN = ?" << outpoint.hash << outpoint.n; + "WHERE txID = ? AND txN = ?" << outpoint.hash << outpoint.n; for (auto&& row: query) { uint32_t coinbase = 0, height = 0; row >> coinbase >> height >> coin.out.nValue >> coin.out.scriptPubKey; @@ -65,7 +65,7 @@ bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { auto query = db << "SELECT 1 FROM unspent " - "WHERE txID = ? and txN = ?" << outpoint.hash << outpoint.n; + "WHERE txID = ? AND txN = ?" << outpoint.hash << outpoint.n; return query.begin() != query.end(); } @@ -110,20 +110,26 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, boo db << "begin"; db << "INSERT OR REPLACE INTO marker VALUES('head_block', ?)" << hashBlock; + + auto dbd = db << "DELETE FROM unspent WHERE txID = ? AND txN = ?"; + auto dbi = db << "INSERT OR REPLACE INTO unspent VALUES(?,?,?,?,?,?,?)"; for (auto it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { if (it->second.coin.IsSpent()) { // at present the "IsSpent" flag is used for both "spent" and "block going backwards" - db << "DELETE FROM unspent WHERE txID = ? AND txN = ?" << it->first.hash << it->first.n; + dbd << it->first.hash << it->first.n; + dbd++; } else { CTxDestination address; std::string destination; if (ExtractDestination(it->second.coin.out.scriptPubKey, address)) destination = EncodeDestination(address); - db << "INSERT OR REPLACE INTO unspent VALUES(?,?,?,?,?,?,?)" << it->first.hash << it->first.n - << it->second.coin.fCoinBase << it->second.coin.nHeight - << it->second.coin.out.nValue << it->second.coin.out.scriptPubKey << destination; + uint32_t isCoinBase = it->second.coin.fCoinBase; // bit-field + uint32_t coinHeight = it->second.coin.nHeight; // bit-field + dbi << it->first.hash << it->first.n << isCoinBase << coinHeight + << it->second.coin.out.nValue << it->second.coin.out.scriptPubKey << destination; + dbi++; } changed++; } @@ -138,6 +144,8 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, boo } } } + dbd.used(true); + dbi.used(true); db << "INSERT OR REPLACE INTO marker VALUES('best_block', ?)" << hashBlock; db << "DELETE FROM marker WHERE name = 'head_block'"; @@ -198,9 +206,6 @@ CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) "nonce INTEGER NOT NULL " ")"; - db << "CREATE UNIQUE INDEX IF NOT EXISTS block_info_height ON block_info (height)"; - db << "CREATE UNIQUE INDEX IF NOT EXISTS block_file_data_pos ON block_info (file, dataPos)"; - db << "CREATE TABLE IF NOT EXISTS tx_to_block (" "txID BLOB NOT NULL PRIMARY KEY, " "file INTEGER NOT NULL, " @@ -217,6 +222,9 @@ CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) db << "DELETE FROM tx_to_block"; db << "DELETE FROM flag"; } + + // not unique because we actually want to store forks: + db << "CREATE INDEX IF NOT EXISTS block_info_height ON block_info (height)"; } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { diff --git a/src/validation.cpp b/src/validation.cpp index 9bd56df5e..2909a7551 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2778,6 +2778,12 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp assert(flushed); flushed = trieCache.flush(); assert(flushed); + +// for verifying that rollback code works: +// auto result = DisconnectBlock(blockConnecting, pindexNew, view, trieCache); +// assert(result == DisconnectResult::DISCONNECT_OK); +// assert(trieCache.getMerkleHash() == pindexNew->pprev->hashClaimTrie); +// LogPrintf("Verified %d!\n", pindexNew->nHeight); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal); diff --git a/src/validation.h b/src/validation.h index 16a4c56e7..b5898da19 100644 --- a/src/validation.h +++ b/src/validation.h @@ -106,7 +106,7 @@ static const int MAX_BLOCKTXN_DEPTH = 10; * want to make this a per-peer adaptive value at some point. */ static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; /** Time to wait (in seconds) between writing blocks/block index to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +static const unsigned int DATABASE_WRITE_INTERVAL = 15 * 60; /** Time to wait (in seconds) between flushing chainstate to disk. */ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */