changed name column to blob

src/claimtrie/sqlite.h

@@ -31,12 +31,9 @@
 
 namespace sqlite
 {
-    template<>
-    struct has_sqlite_type<CUint256, SQLITE_BLOB, void> : std::true_type {};
 
     template<>
     struct has_sqlite_type<CUint160, SQLITE_BLOB, void> : std::true_type {};
-
     inline CUint160 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<CUint160>) {
         CUint160 ret;
         auto ptr = sqlite3_column_blob(stmt, inx);
@@ -47,6 +44,8 @@ namespace sqlite
         return ret;
     }
 
+    template<>
+    struct has_sqlite_type<CUint256, SQLITE_BLOB, void> : std::true_type {};
     inline CUint256 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<CUint256>) {
         CUint256 ret;
         auto ptr = sqlite3_column_blob(stmt, inx);

src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h

@@ -421,12 +421,12 @@ namespace sqlite {
         }
 
         template <typename Function>
-        void define(const std::string &name, Function&& func) {
+        void define(const std::string &name, Function&& func, bool deterministic = true) {
             typedef utility::function_traits<Function> traits;
 
             auto funcPtr = new auto(std::forward<Function>(func));
             if(int result = sqlite3_create_function_v2(
-                    _db.get(), name.data(), traits::arity, SQLITE_UTF8, funcPtr,
+                    _db.get(), name.data(), traits::arity, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr,
                     sql_function_binder::scalar<traits::arity, typename std::remove_reference<Function>::type>,
                     nullptr, nullptr, [](void* ptr){
                         delete static_cast<decltype(funcPtr)>(ptr);
@@ -435,13 +435,13 @@ namespace sqlite {
         }
 
         template <typename StepFunction, typename FinalFunction>
-        void define(const std::string &name, StepFunction&& step, FinalFunction&& final) {
+        void define(const std::string &name, StepFunction&& step, FinalFunction&& final, bool deterministic = true) {
             typedef utility::function_traits<StepFunction> traits;
             using ContextType = typename std::remove_reference<typename traits::template argument<0>>::type;
 
             auto funcPtr = new auto(std::make_pair(std::forward<StepFunction>(step), std::forward<FinalFunction>(final)));
             if(int result = sqlite3_create_function_v2(
-                    _db.get(), name.c_str(), traits::arity - 1, SQLITE_UTF8, funcPtr, nullptr,
+                    _db.get(), name.c_str(), traits::arity - 1, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr, nullptr,
                     sql_function_binder::step<ContextType, traits::arity, typename std::remove_reference<decltype(*funcPtr)>::type>,
                     sql_function_binder::final<ContextType, typename std::remove_reference<decltype(*funcPtr)>::type>,
                     [](void* ptr){

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/type_wrapper.h

@@ -180,27 +180,47 @@ namespace sqlite {
 
     // str_ref
     template<>
-    struct has_sqlite_type<std::string, SQLITE3_TEXT, void> : std::true_type {};
+    struct has_sqlite_type<std::string, SQLITE_BLOB, void> : std::true_type {}; //
     inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, str_ref val) {
-        return sqlite3_bind_text(stmt, inx, val.data(), val.length(), SQLITE_TRANSIENT);
+        return sqlite3_bind_blob(stmt, inx, val.data(), val.length(), SQLITE_TRANSIENT);
     }
 
     // Convert char* to string_view to trigger op<<(..., const str_ref )
     template<std::size_t N> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const char(&STR)[N]) {
-        return sqlite3_bind_text(stmt, inx, &STR[0], N-1, SQLITE_TRANSIENT);
+        return sqlite3_bind_blob(stmt, inx, &STR[0], N-1, SQLITE_TRANSIENT);
     }
 
     inline std::string get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::string>) {
-        return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? std::string() :
+        auto type = sqlite3_column_type(stmt, inx);
-               std::string(reinterpret_cast<char const *>(sqlite3_column_text(stmt, inx)), sqlite3_column_bytes(stmt, inx));
+        switch (type) {
+            case SQLITE_INTEGER:
+                return std::to_string(sqlite3_column_int64(stmt, inx));
+            case SQLITE_FLOAT:
+                return std::to_string(sqlite3_column_double(stmt, inx));
+            case SQLITE_BLOB: // It's important to support both text and blob data as txdb has some text and trie has some blob
+                return std::string(reinterpret_cast<char const *>(sqlite3_column_blob(stmt, inx)), sqlite3_column_bytes(stmt, inx));
+            case SQLITE3_TEXT:
+                return std::string(reinterpret_cast<char const *>(sqlite3_column_text(stmt, inx)), sqlite3_column_bytes(stmt, inx));
+        }
+        return std::string(); // NULL
     }
-    inline std::string  get_val_from_db(sqlite3_value *value, result_type<std::string >) {
+    inline std::string get_val_from_db(sqlite3_value *value, result_type<std::string >) {
-        return sqlite3_value_type(value) == SQLITE_NULL ? std::string() :
+        auto type = sqlite3_value_type(value);
-               std::string(reinterpret_cast<char const *>(sqlite3_value_text(value)), sqlite3_value_bytes(value));
+        switch (type) {
+            case SQLITE_INTEGER:
+                return std::to_string(sqlite3_value_int64(value));
+            case SQLITE_FLOAT:
+                return std::to_string(sqlite3_value_double(value));
+            case SQLITE_BLOB:
+                return std::string(reinterpret_cast<char const *>(sqlite3_value_blob(value)), sqlite3_value_bytes(value));
+            case SQLITE3_TEXT:
+                return std::string(reinterpret_cast<char const *>(sqlite3_value_text(value)), sqlite3_value_bytes(value));
+        }
+        return std::string(); // NULL
     }
 
     inline void store_result_in_db(sqlite3_context* db, str_ref val) {
-        sqlite3_result_text(db, val.data(), val.length(), SQLITE_TRANSIENT);
+        sqlite3_result_blob(db, val.data(), val.length(), SQLITE_TRANSIENT);
     }
     // u16str_ref
     template<>

src/claimtrie/sqlite/sqlite3.h

@@ -41,6 +41,8 @@
 extern "C" {
 #endif
 
+#define SQLITE_OMIT_COMPILEOPTION_DIAGS 1
+#define SQLITE_OMIT_DEPRECATED 1
 
 /*
 ** Provide the ability to override linkage features of the interface.

src/claimtrie/trie.cpp

@@ -66,22 +66,22 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
 {
     applyPragmas(db, 5U * 1024U); // in KB
 
-    db << "CREATE TABLE IF NOT EXISTS node (name TEXT NOT NULL PRIMARY KEY, "
+    db << "CREATE TABLE IF NOT EXISTS node (name BLOB NOT NULL PRIMARY KEY, "
-          "parent TEXT REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
+          "parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
-          "hash BLOB COLLATE BINARY)";
+          "hash BLOB)";
 
     db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)";
     // db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain
     db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)";
 
-    db << "CREATE TABLE IF NOT EXISTS takeover (name TEXT NOT NULL, height INTEGER NOT NULL, "
+    db << "CREATE TABLE IF NOT EXISTS takeover (name BLOB NOT NULL, height INTEGER NOT NULL, "
-           "claimID BLOB COLLATE BINARY, PRIMARY KEY(name, height DESC));";
+           "claimID BLOB, PRIMARY KEY(name, height DESC));";
 
     db << "CREATE INDEX IF NOT EXISTS takeover_height ON takeover (height)";
 
-    db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL COLLATE BINARY PRIMARY KEY, name TEXT NOT NULL, "
+    db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL PRIMARY KEY, name BLOB NOT NULL, "
-           "nodeName TEXT NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
+           "nodeName BLOB NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
-           "txID BLOB NOT NULL COLLATE BINARY, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, "
+           "txID BLOB NOT NULL, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, "
            "validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, "
            "expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL);";
 
@@ -89,8 +89,8 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
     db << "CREATE INDEX IF NOT EXISTS claim_expirationHeight ON claim (expirationHeight)";
     db << "CREATE INDEX IF NOT EXISTS claim_nodeName ON claim (nodeName)";
 
-    db << "CREATE TABLE IF NOT EXISTS support (txID BLOB NOT NULL COLLATE BINARY, txN INTEGER NOT NULL, "
+    db << "CREATE TABLE IF NOT EXISTS support (txID BLOB NOT NULL, txN INTEGER NOT NULL, "
-           "supportedClaimID BLOB NOT NULL COLLATE BINARY, name TEXT NOT NULL, nodeName TEXT NOT NULL, "
+           "supportedClaimID BLOB NOT NULL, name BLOB NOT NULL, nodeName BLOB NOT NULL, "
            "blockHeight INTEGER NOT NULL, validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, "
            "expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL, PRIMARY KEY(txID, txN));";
 
@@ -106,7 +106,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
         db << "DELETE FROM takeover";
     }
 
-    db << "INSERT OR IGNORE INTO node(name, hash) VALUES('', ?)" << one; // ensure that we always have our root node
+    db << "INSERT OR IGNORE INTO node(name, hash) VALUES(x'', ?)" << one; // ensure that we always have our root node
 }
 
 CClaimTrieCacheBase::~CClaimTrieCacheBase()
@@ -245,7 +245,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate()
     // assume parents are not set correctly here:
     auto parentQuery = db << "SELECT MAX(name) FROM node WHERE "
                               "name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
-                              "SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix)";
+                              "SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix)";
 
     auto insertQuery = db << "INSERT INTO node(name, parent, hash) VALUES(?, ?, NULL) "
                              "ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL";
@@ -310,7 +310,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate()
     // parents should all be set right
     db << "UPDATE node SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS "
           "(SELECT parent FROM node WHERE hash IS NULL UNION SELECT parent FROM prefix, node "
-          "WHERE name = prefix.p AND prefix.p != '') SELECT p FROM prefix)";
+          "WHERE name = prefix.p AND prefix.p != x'') SELECT p FROM prefix)";
 }
 
 std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const
@@ -494,7 +494,7 @@ extern const std::string proofClaimQuery_s =
     "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
     "FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM node n "
     "WHERE n.name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
-    "SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix) "
+    "SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix) "
     "ORDER BY n.name";
 
 CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
@@ -507,7 +507,6 @@ CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
     nNextHeight = base->nNextHeight;
 
     applyPragmas(db, base->dbCacheBytes >> 10U); // in KB
-
     db.define("POPS", [](std::string s) -> std::string { if (!s.empty()) s.pop_back(); return s; });
 }
 
@@ -528,7 +527,7 @@ CUint256 CClaimTrieCacheBase::getMerkleHash()
 {
     ensureTreeStructureIsUpToDate();
     CUint256 hash;
-    db  << "SELECT hash FROM node WHERE name = ''"
+    db  << "SELECT hash FROM node WHERE name = x''"
         >> [&hash](std::unique_ptr<CUint256> rootHash) {
             if (rootHash)
                 hash = std::move(*rootHash);
@@ -538,7 +537,7 @@ CUint256 CClaimTrieCacheBase::getMerkleHash()
     assert(transacting); // no data changed but we didn't have the root hash there already?
     auto updateQuery = db << "UPDATE node SET hash = ? WHERE name = ?";
     db << "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END FROM takeover t WHERE t.name = n.name "
-            "ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC"
+            "ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob
         >> [this, &hash, &updateQuery](const std::string& name, int takeoverHeight) {
             hash = computeNodeHash(name, takeoverHeight);
             updateQuery << hash << name;
@@ -640,15 +639,24 @@ bool CClaimTrieCacheBase::removeClaim(const CUint160& claimId, const CTxOutPoint
     // when node should be deleted from cache but instead it's kept
     // because it's a parent one and should not be effectively erased
     // we had a bug in the old code where that situation would force a zero delay on re-add
-    if (true) { // TODO: hard fork this out (which we already tried once but failed)
+    if (nNextHeight >= 297706) { // TODO: hard fork this out (which we already tried once but failed)
-        const static std::string charsThatBreakLikeOp("_%\0", 3);
+        // we have to jump through some hoops here to make the claim_nodeName index be used on partial blobs
-        bool cantUseLike = nodeName.find_first_of(charsThatBreakLikeOp) != std::string::npos;
+        // neither LIKE nor SUBSTR will use an index on a blob (which is lame because it would be simple)
-        auto query = cantUseLike ? (db << "SELECT nodeName FROM claim WHERE SUBSTR(nodeName, 1, ?3) = ?1 "
+        auto end = nodeName;
+        auto usingRange = false;
+        if (!end.empty() && end.back() < std::numeric_limits<char>::max()) {
+            ++end.back(); // the fast path
+            usingRange = true;
+        }
+        // else
+        //    end += std::string(256U, std::numeric_limits<char>::max()); // 256 is our supposed max length claim, but that's not enforced anywhere presently
+        auto query = usingRange ?
+                     (db << "SELECT nodeName FROM claim WHERE nodeName BETWEEN ?1 AND ?2 "
+                            "AND nodeName != ?2 AND activationHeight < ?3 AND expirationHeight > ?3 ORDER BY nodeName LIMIT 1"
+                         << nodeName << end << nNextHeight) :
+                    (db << "SELECT nodeName FROM claim INDEXED BY claim_nodeName WHERE SUBSTR(nodeName, 1, ?3) = ?1 "
                            "AND activationHeight < ?2 AND expirationHeight > ?2 ORDER BY nodeName LIMIT 1"
-                            << nodeName << nNextHeight << nodeName.size()) :
+                            << nodeName << nNextHeight << nodeName.size());
-                        (db << "SELECT nodeName FROM claim WHERE nodeName LIKE ?1 "
-                            "AND activationHeight < ?2 AND expirationHeight > ?2 ORDER BY nodeName LIMIT 1"
-                            << nodeName + '%' << nNextHeight);
         for (auto&& row: query) {
             std::string shortestMatch;
             row >> shortestMatch;