From 5d5f09bafb49be2cb1a37f2482b356cb84ce0417 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Fri, 13 Dec 2019 19:20:48 -0700 Subject: [PATCH] changed name column to blob --- src/claimtrie/sqlite.h | 5 +- src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h | 8 +-- .../hdr/sqlite_modern_cpp/type_wrapper.h | 38 +++++++++--- src/claimtrie/sqlite/sqlite3.h | 2 + src/claimtrie/trie.cpp | 58 +++++++++++-------- 5 files changed, 70 insertions(+), 41 deletions(-) diff --git a/src/claimtrie/sqlite.h b/src/claimtrie/sqlite.h index f309ae822..74af4f733 100644 --- a/src/claimtrie/sqlite.h +++ b/src/claimtrie/sqlite.h @@ -31,12 +31,9 @@ namespace sqlite { - template<> - struct has_sqlite_type : std::true_type {}; template<> struct has_sqlite_type : std::true_type {}; - inline CUint160 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type) { CUint160 ret; auto ptr = sqlite3_column_blob(stmt, inx); @@ -47,6 +44,8 @@ namespace sqlite return ret; } + template<> + struct has_sqlite_type : std::true_type {}; inline CUint256 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type) { CUint256 ret; auto ptr = sqlite3_column_blob(stmt, inx); diff --git a/src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h b/src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h index 1e419b1ee..81721d0cb 100644 --- a/src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h +++ b/src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h @@ -421,12 +421,12 @@ namespace sqlite { } template - void define(const std::string &name, Function&& func) { + void define(const std::string &name, Function&& func, bool deterministic = true) { typedef utility::function_traits traits; auto funcPtr = new auto(std::forward(func)); if(int result = sqlite3_create_function_v2( - _db.get(), name.data(), traits::arity, SQLITE_UTF8, funcPtr, + _db.get(), name.data(), traits::arity, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr, sql_function_binder::scalar::type>, nullptr, nullptr, [](void* ptr){ delete static_cast(ptr); @@ -435,13 +435,13 @@ namespace sqlite { } template - void define(const std::string &name, StepFunction&& step, FinalFunction&& final) { + void define(const std::string &name, StepFunction&& step, FinalFunction&& final, bool deterministic = true) { typedef utility::function_traits traits; using ContextType = typename std::remove_reference>::type; auto funcPtr = new auto(std::make_pair(std::forward(step), std::forward(final))); if(int result = sqlite3_create_function_v2( - _db.get(), name.c_str(), traits::arity - 1, SQLITE_UTF8, funcPtr, nullptr, + _db.get(), name.c_str(), traits::arity - 1, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr, nullptr, sql_function_binder::step::type>, sql_function_binder::final::type>, [](void* ptr){ diff --git a/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/type_wrapper.h b/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/type_wrapper.h index b93d13a15..b915ef83e 100644 --- a/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/type_wrapper.h +++ b/src/claimtrie/sqlite/hdr/sqlite_modern_cpp/type_wrapper.h @@ -180,27 +180,47 @@ namespace sqlite { // str_ref template<> - struct has_sqlite_type : std::true_type {}; + struct has_sqlite_type : std::true_type {}; // inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, str_ref val) { - return sqlite3_bind_text(stmt, inx, val.data(), val.length(), SQLITE_TRANSIENT); + return sqlite3_bind_blob(stmt, inx, val.data(), val.length(), SQLITE_TRANSIENT); } // Convert char* to string_view to trigger op<<(..., const str_ref ) template inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const char(&STR)[N]) { - return sqlite3_bind_text(stmt, inx, &STR[0], N-1, SQLITE_TRANSIENT); + return sqlite3_bind_blob(stmt, inx, &STR[0], N-1, SQLITE_TRANSIENT); } inline std::string get_col_from_db(sqlite3_stmt* stmt, int inx, result_type) { - return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? std::string() : - std::string(reinterpret_cast(sqlite3_column_text(stmt, inx)), sqlite3_column_bytes(stmt, inx)); + auto type = sqlite3_column_type(stmt, inx); + switch (type) { + case SQLITE_INTEGER: + return std::to_string(sqlite3_column_int64(stmt, inx)); + case SQLITE_FLOAT: + return std::to_string(sqlite3_column_double(stmt, inx)); + case SQLITE_BLOB: // It's important to support both text and blob data as txdb has some text and trie has some blob + return std::string(reinterpret_cast(sqlite3_column_blob(stmt, inx)), sqlite3_column_bytes(stmt, inx)); + case SQLITE3_TEXT: + return std::string(reinterpret_cast(sqlite3_column_text(stmt, inx)), sqlite3_column_bytes(stmt, inx)); + } + return std::string(); // NULL } - inline std::string get_val_from_db(sqlite3_value *value, result_type) { - return sqlite3_value_type(value) == SQLITE_NULL ? std::string() : - std::string(reinterpret_cast(sqlite3_value_text(value)), sqlite3_value_bytes(value)); + inline std::string get_val_from_db(sqlite3_value *value, result_type) { + auto type = sqlite3_value_type(value); + switch (type) { + case SQLITE_INTEGER: + return std::to_string(sqlite3_value_int64(value)); + case SQLITE_FLOAT: + return std::to_string(sqlite3_value_double(value)); + case SQLITE_BLOB: + return std::string(reinterpret_cast(sqlite3_value_blob(value)), sqlite3_value_bytes(value)); + case SQLITE3_TEXT: + return std::string(reinterpret_cast(sqlite3_value_text(value)), sqlite3_value_bytes(value)); + } + return std::string(); // NULL } inline void store_result_in_db(sqlite3_context* db, str_ref val) { - sqlite3_result_text(db, val.data(), val.length(), SQLITE_TRANSIENT); + sqlite3_result_blob(db, val.data(), val.length(), SQLITE_TRANSIENT); } // u16str_ref template<> diff --git a/src/claimtrie/sqlite/sqlite3.h b/src/claimtrie/sqlite/sqlite3.h index 37bfac528..08d283c51 100644 --- a/src/claimtrie/sqlite/sqlite3.h +++ b/src/claimtrie/sqlite/sqlite3.h @@ -41,6 +41,8 @@ extern "C" { #endif +#define SQLITE_OMIT_COMPILEOPTION_DIAGS 1 +#define SQLITE_OMIT_DEPRECATED 1 /* ** Provide the ability to override linkage features of the interface. diff --git a/src/claimtrie/trie.cpp b/src/claimtrie/trie.cpp index e3fff6602..6fe09a347 100644 --- a/src/claimtrie/trie.cpp +++ b/src/claimtrie/trie.cpp @@ -66,22 +66,22 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, { applyPragmas(db, 5U * 1024U); // in KB - db << "CREATE TABLE IF NOT EXISTS node (name TEXT NOT NULL PRIMARY KEY, " - "parent TEXT REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " - "hash BLOB COLLATE BINARY)"; + db << "CREATE TABLE IF NOT EXISTS node (name BLOB NOT NULL PRIMARY KEY, " + "parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " + "hash BLOB)"; db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)"; // db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)"; - db << "CREATE TABLE IF NOT EXISTS takeover (name TEXT NOT NULL, height INTEGER NOT NULL, " - "claimID BLOB COLLATE BINARY, PRIMARY KEY(name, height DESC));"; + db << "CREATE TABLE IF NOT EXISTS takeover (name BLOB NOT NULL, height INTEGER NOT NULL, " + "claimID BLOB, PRIMARY KEY(name, height DESC));"; db << "CREATE INDEX IF NOT EXISTS takeover_height ON takeover (height)"; - db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL COLLATE BINARY PRIMARY KEY, name TEXT NOT NULL, " - "nodeName TEXT NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " - "txID BLOB NOT NULL COLLATE BINARY, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, " + db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL PRIMARY KEY, name BLOB NOT NULL, " + "nodeName BLOB NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, " + "txID BLOB NOT NULL, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, " "validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, " "expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL);"; @@ -89,8 +89,8 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, db << "CREATE INDEX IF NOT EXISTS claim_expirationHeight ON claim (expirationHeight)"; db << "CREATE INDEX IF NOT EXISTS claim_nodeName ON claim (nodeName)"; - db << "CREATE TABLE IF NOT EXISTS support (txID BLOB NOT NULL COLLATE BINARY, txN INTEGER NOT NULL, " - "supportedClaimID BLOB NOT NULL COLLATE BINARY, name TEXT NOT NULL, nodeName TEXT NOT NULL, " + db << "CREATE TABLE IF NOT EXISTS support (txID BLOB NOT NULL, txN INTEGER NOT NULL, " + "supportedClaimID BLOB NOT NULL, name BLOB NOT NULL, nodeName BLOB NOT NULL, " "blockHeight INTEGER NOT NULL, validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, " "expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL, PRIMARY KEY(txID, txN));"; @@ -106,7 +106,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height, db << "DELETE FROM takeover"; } - db << "INSERT OR IGNORE INTO node(name, hash) VALUES('', ?)" << one; // ensure that we always have our root node + db << "INSERT OR IGNORE INTO node(name, hash) VALUES(x'', ?)" << one; // ensure that we always have our root node } CClaimTrieCacheBase::~CClaimTrieCacheBase() @@ -245,7 +245,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() // assume parents are not set correctly here: auto parentQuery = db << "SELECT MAX(name) FROM node WHERE " "name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " - "SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix)"; + "SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix)"; auto insertQuery = db << "INSERT INTO node(name, parent, hash) VALUES(?, ?, NULL) " "ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL"; @@ -310,7 +310,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate() // parents should all be set right db << "UPDATE node SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS " "(SELECT parent FROM node WHERE hash IS NULL UNION SELECT parent FROM prefix, node " - "WHERE name = prefix.p AND prefix.p != '') SELECT p FROM prefix)"; + "WHERE name = prefix.p AND prefix.p != x'') SELECT p FROM prefix)"; } std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const @@ -494,7 +494,7 @@ extern const std::string proofClaimQuery_s = "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END " "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM node n " "WHERE n.name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL " - "SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix) " + "SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix) " "ORDER BY n.name"; CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) @@ -507,7 +507,6 @@ CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) nNextHeight = base->nNextHeight; applyPragmas(db, base->dbCacheBytes >> 10U); // in KB - db.define("POPS", [](std::string s) -> std::string { if (!s.empty()) s.pop_back(); return s; }); } @@ -528,7 +527,7 @@ CUint256 CClaimTrieCacheBase::getMerkleHash() { ensureTreeStructureIsUpToDate(); CUint256 hash; - db << "SELECT hash FROM node WHERE name = ''" + db << "SELECT hash FROM node WHERE name = x''" >> [&hash](std::unique_ptr rootHash) { if (rootHash) hash = std::move(*rootHash); @@ -538,7 +537,7 @@ CUint256 CClaimTrieCacheBase::getMerkleHash() assert(transacting); // no data changed but we didn't have the root hash there already? auto updateQuery = db << "UPDATE node SET hash = ? WHERE name = ?"; db << "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END FROM takeover t WHERE t.name = n.name " - "ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" + "ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob >> [this, &hash, &updateQuery](const std::string& name, int takeoverHeight) { hash = computeNodeHash(name, takeoverHeight); updateQuery << hash << name; @@ -640,15 +639,24 @@ bool CClaimTrieCacheBase::removeClaim(const CUint160& claimId, const CTxOutPoint // when node should be deleted from cache but instead it's kept // because it's a parent one and should not be effectively erased // we had a bug in the old code where that situation would force a zero delay on re-add - if (true) { // TODO: hard fork this out (which we already tried once but failed) - const static std::string charsThatBreakLikeOp("_%\0", 3); - bool cantUseLike = nodeName.find_first_of(charsThatBreakLikeOp) != std::string::npos; - auto query = cantUseLike ? (db << "SELECT nodeName FROM claim WHERE SUBSTR(nodeName, 1, ?3) = ?1 " + if (nNextHeight >= 297706) { // TODO: hard fork this out (which we already tried once but failed) + // we have to jump through some hoops here to make the claim_nodeName index be used on partial blobs + // neither LIKE nor SUBSTR will use an index on a blob (which is lame because it would be simple) + auto end = nodeName; + auto usingRange = false; + if (!end.empty() && end.back() < std::numeric_limits::max()) { + ++end.back(); // the fast path + usingRange = true; + } + // else + // end += std::string(256U, std::numeric_limits::max()); // 256 is our supposed max length claim, but that's not enforced anywhere presently + auto query = usingRange ? + (db << "SELECT nodeName FROM claim WHERE nodeName BETWEEN ?1 AND ?2 " + "AND nodeName != ?2 AND activationHeight < ?3 AND expirationHeight > ?3 ORDER BY nodeName LIMIT 1" + << nodeName << end << nNextHeight) : + (db << "SELECT nodeName FROM claim INDEXED BY claim_nodeName WHERE SUBSTR(nodeName, 1, ?3) = ?1 " "AND activationHeight < ?2 AND expirationHeight > ?2 ORDER BY nodeName LIMIT 1" - << nodeName << nNextHeight << nodeName.size()) : - (db << "SELECT nodeName FROM claim WHERE nodeName LIKE ?1 " - "AND activationHeight < ?2 AND expirationHeight > ?2 ORDER BY nodeName LIMIT 1" - << nodeName + '%' << nNextHeight); + << nodeName << nNextHeight << nodeName.size()); for (auto&& row: query) { std::string shortestMatch; row >> shortestMatch;