LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

changed name column to blob

Browse source
This commit is contained in:
Brannon King 2019-12-13 19:20:48 -07:00 committed by Anthony Fieroni
parent 38006baf03
commit 8ddb17b03d
5 changed files with 70 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/claimtrie/sqlite.h
View file

@ -31,12 +31,9 @@
namespace sqlite
{
template<>
struct has_sqlite_type<CUint256, SQLITE_BLOB, void> : std::true_type {};
template<>
struct has_sqlite_type<CUint160, SQLITE_BLOB, void> : std::true_type {};
inline CUint160 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<CUint160>) {
CUint160 ret;
auto ptr = sqlite3_column_blob(stmt, inx);
@ -47,6 +44,8 @@ namespace sqlite
return ret;
}
template<>
struct has_sqlite_type<CUint256, SQLITE_BLOB, void> : std::true_type {};
inline CUint256 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<CUint256>) {
CUint256 ret;
auto ptr = sqlite3_column_blob(stmt, inx);

8
src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h
View file

@ -421,12 +421,12 @@ namespace sqlite {
}
template <typename Function>
void define(const std::string &name, Function&& func) {
void define(const std::string &name, Function&& func, bool deterministic = true) {
typedef utility::function_traits<Function> traits;
auto funcPtr = new auto(std::forward<Function>(func));
if(int result = sqlite3_create_function_v2(
_db.get(), name.data(), traits::arity, SQLITE_UTF8, funcPtr,
_db.get(), name.data(), traits::arity, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr,
sql_function_binder::scalar<traits::arity, typename std::remove_reference<Function>::type>,
nullptr, nullptr, [](void* ptr){
delete static_cast<decltype(funcPtr)>(ptr);
@ -435,13 +435,13 @@ namespace sqlite {
}
template <typename StepFunction, typename FinalFunction>
void define(const std::string &name, StepFunction&& step, FinalFunction&& final) {
void define(const std::string &name, StepFunction&& step, FinalFunction&& final, bool deterministic = true) {
typedef utility::function_traits<StepFunction> traits;
using ContextType = typename std::remove_reference<typename traits::template argument<0>>::type;
auto funcPtr = new auto(std::make_pair(std::forward<StepFunction>(step), std::forward<FinalFunction>(final)));
if(int result = sqlite3_create_function_v2(
_db.get(), name.c_str(), traits::arity - 1, SQLITE_UTF8, funcPtr, nullptr,
_db.get(), name.c_str(), traits::arity - 1, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr, nullptr,
sql_function_binder::step<ContextType, traits::arity, typename std::remove_reference<decltype(*funcPtr)>::type>,
sql_function_binder::final<ContextType, typename std::remove_reference<decltype(*funcPtr)>::type>,
[](void* ptr){

38
src/claimtrie/sqlite/hdr/sqlite_modern_cpp/type_wrapper.h
View file

@ -180,27 +180,47 @@ namespace sqlite {
// str_ref
template<>
struct has_sqlite_type<std::string, SQLITE3_TEXT, void> : std::true_type {};
struct has_sqlite_type<std::string, SQLITE_BLOB, void> : std::true_type {}; //
inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, str_ref val) {
return sqlite3_bind_text(stmt, inx, val.data(), val.length(), SQLITE_TRANSIENT);
return sqlite3_bind_blob(stmt, inx, val.data(), val.length(), SQLITE_TRANSIENT);
}
// Convert char* to string_view to trigger op<<(..., const str_ref )
template<std::size_t N> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const char(&STR)[N]) {
return sqlite3_bind_text(stmt, inx, &STR[0], N-1, SQLITE_TRANSIENT);
return sqlite3_bind_blob(stmt, inx, &STR[0], N-1, SQLITE_TRANSIENT);
}
inline std::string get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::string>) {
return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? std::string() :
std::string(reinterpret_cast<char const *>(sqlite3_column_text(stmt, inx)), sqlite3_column_bytes(stmt, inx));
auto type = sqlite3_column_type(stmt, inx);
switch (type) {
case SQLITE_INTEGER:
return std::to_string(sqlite3_column_int64(stmt, inx));
case SQLITE_FLOAT:
return std::to_string(sqlite3_column_double(stmt, inx));
case SQLITE_BLOB: // It's important to support both text and blob data as txdb has some text and trie has some blob
return std::string(reinterpret_cast<char const *>(sqlite3_column_blob(stmt, inx)), sqlite3_column_bytes(stmt, inx));
case SQLITE3_TEXT:
return std::string(reinterpret_cast<char const *>(sqlite3_column_text(stmt, inx)), sqlite3_column_bytes(stmt, inx));
}
return std::string(); // NULL
}
inline std::string get_val_from_db(sqlite3_value *value, result_type<std::string >) {
return sqlite3_value_type(value) == SQLITE_NULL ? std::string() :
std::string(reinterpret_cast<char const *>(sqlite3_value_text(value)), sqlite3_value_bytes(value));
inline std::string get_val_from_db(sqlite3_value *value, result_type<std::string >) {
auto type = sqlite3_value_type(value);
switch (type) {
case SQLITE_INTEGER:
return std::to_string(sqlite3_value_int64(value));
case SQLITE_FLOAT:
return std::to_string(sqlite3_value_double(value));
case SQLITE_BLOB:
return std::string(reinterpret_cast<char const *>(sqlite3_value_blob(value)), sqlite3_value_bytes(value));
case SQLITE3_TEXT:
return std::string(reinterpret_cast<char const *>(sqlite3_value_text(value)), sqlite3_value_bytes(value));
}
return std::string(); // NULL
}
inline void store_result_in_db(sqlite3_context* db, str_ref val) {
sqlite3_result_text(db, val.data(), val.length(), SQLITE_TRANSIENT);
sqlite3_result_blob(db, val.data(), val.length(), SQLITE_TRANSIENT);
}
// u16str_ref
template<>

2
src/claimtrie/sqlite/sqlite3.h
View file

@ -41,6 +41,8 @@
extern "C" {
#endif
#define SQLITE_OMIT_COMPILEOPTION_DIAGS 1
#define SQLITE_OMIT_DEPRECATED 1
/*
** Provide the ability to override linkage features of the interface.

58
src/claimtrie/trie.cpp
View file

@ -66,22 +66,22 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
{
applyPragmas(db, 5U * 1024U); // in KB
db << "CREATE TABLE IF NOT EXISTS node (name TEXT NOT NULL PRIMARY KEY, "
"parent TEXT REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"hash BLOB COLLATE BINARY)";
db << "CREATE TABLE IF NOT EXISTS node (name BLOB NOT NULL PRIMARY KEY, "
"parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"hash BLOB)";
db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)";
// db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain
db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)";
db << "CREATE TABLE IF NOT EXISTS takeover (name TEXT NOT NULL, height INTEGER NOT NULL, "
"claimID BLOB COLLATE BINARY, PRIMARY KEY(name, height DESC));";
db << "CREATE TABLE IF NOT EXISTS takeover (name BLOB NOT NULL, height INTEGER NOT NULL, "
"claimID BLOB, PRIMARY KEY(name, height DESC));";
db << "CREATE INDEX IF NOT EXISTS takeover_height ON takeover (height)";
db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL COLLATE BINARY PRIMARY KEY, name TEXT NOT NULL, "
"nodeName TEXT NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"txID BLOB NOT NULL COLLATE BINARY, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, "
db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL PRIMARY KEY, name BLOB NOT NULL, "
"nodeName BLOB NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"txID BLOB NOT NULL, txN INTEGER NOT NULL, blockHeight INTEGER NOT NULL, "
"validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, "
"expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL);";
@ -89,8 +89,8 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
db << "CREATE INDEX IF NOT EXISTS claim_expirationHeight ON claim (expirationHeight)";
db << "CREATE INDEX IF NOT EXISTS claim_nodeName ON claim (nodeName)";
db << "CREATE TABLE IF NOT EXISTS support (txID BLOB NOT NULL COLLATE BINARY, txN INTEGER NOT NULL, "
"supportedClaimID BLOB NOT NULL COLLATE BINARY, name TEXT NOT NULL, nodeName TEXT NOT NULL, "
db << "CREATE TABLE IF NOT EXISTS support (txID BLOB NOT NULL, txN INTEGER NOT NULL, "
"supportedClaimID BLOB NOT NULL, name BLOB NOT NULL, nodeName BLOB NOT NULL, "
"blockHeight INTEGER NOT NULL, validHeight INTEGER NOT NULL, activationHeight INTEGER NOT NULL, "
"expirationHeight INTEGER NOT NULL, amount INTEGER NOT NULL, PRIMARY KEY(txID, txN));";
@ -106,7 +106,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
db << "DELETE FROM takeover";
}
db << "INSERT OR IGNORE INTO node(name, hash) VALUES('', ?)" << one; // ensure that we always have our root node
db << "INSERT OR IGNORE INTO node(name, hash) VALUES(x'', ?)" << one; // ensure that we always have our root node
}
CClaimTrieCacheBase::~CClaimTrieCacheBase()
@ -245,7 +245,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate()
// assume parents are not set correctly here:
auto parentQuery = db << "SELECT MAX(name) FROM node WHERE "
"name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
"SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix)";
"SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix)";
auto insertQuery = db << "INSERT INTO node(name, parent, hash) VALUES(?, ?, NULL) "
"ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL";
@ -310,7 +310,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate()
// parents should all be set right
db << "UPDATE node SET hash = NULL WHERE name IN (WITH RECURSIVE prefix(p) AS "
"(SELECT parent FROM node WHERE hash IS NULL UNION SELECT parent FROM prefix, node "
"WHERE name = prefix.p AND prefix.p != '') SELECT p FROM prefix)";
"WHERE name = prefix.p AND prefix.p != x'') SELECT p FROM prefix)";
}
std::size_t CClaimTrieCacheBase::getTotalNamesInTrie() const
@ -494,7 +494,7 @@ extern const std::string proofClaimQuery_s =
"SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM node n "
"WHERE n.name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
"SELECT POPS(p) FROM prefix WHERE p != '') SELECT p FROM prefix) "
"SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix) "
"ORDER BY n.name";
CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
@ -507,7 +507,6 @@ CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
nNextHeight = base->nNextHeight;
applyPragmas(db, base->dbCacheBytes >> 10U); // in KB
db.define("POPS", [](std::string s) -> std::string { if (!s.empty()) s.pop_back(); return s; });
}
@ -528,7 +527,7 @@ CUint256 CClaimTrieCacheBase::getMerkleHash()
{
ensureTreeStructureIsUpToDate();
CUint256 hash;
db << "SELECT hash FROM node WHERE name = ''"
db << "SELECT hash FROM node WHERE name = x''"
>> [&hash](std::unique_ptr<CUint256> rootHash) {
if (rootHash)
hash = std::move(*rootHash);
@ -538,7 +537,7 @@ CUint256 CClaimTrieCacheBase::getMerkleHash()
assert(transacting); // no data changed but we didn't have the root hash there already?
auto updateQuery = db << "UPDATE node SET hash = ? WHERE name = ?";
db << "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END FROM takeover t WHERE t.name = n.name "
"ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC"
"ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob
>> [this, &hash, &updateQuery](const std::string& name, int takeoverHeight) {
hash = computeNodeHash(name, takeoverHeight);
updateQuery << hash << name;
@ -640,15 +639,24 @@ bool CClaimTrieCacheBase::removeClaim(const CUint160& claimId, const CTxOutPoint
// when node should be deleted from cache but instead it's kept
// because it's a parent one and should not be effectively erased
// we had a bug in the old code where that situation would force a zero delay on re-add
if (true) { // TODO: hard fork this out (which we already tried once but failed)
const static std::string charsThatBreakLikeOp("_%\0", 3);
bool cantUseLike = nodeName.find_first_of(charsThatBreakLikeOp) != std::string::npos;
auto query = cantUseLike ? (db << "SELECT nodeName FROM claim WHERE SUBSTR(nodeName, 1, ?3) = ?1 "
if (nNextHeight >= 297706) { // TODO: hard fork this out (which we already tried once but failed)
// we have to jump through some hoops here to make the claim_nodeName index be used on partial blobs
// neither LIKE nor SUBSTR will use an index on a blob (which is lame because it would be simple)
auto end = nodeName;
auto usingRange = false;
if (!end.empty() && end.back() < std::numeric_limits<char>::max()) {
++end.back(); // the fast path
usingRange = true;
}
// else
// end += std::string(256U, std::numeric_limits<char>::max()); // 256 is our supposed max length claim, but that's not enforced anywhere presently
auto query = usingRange ?
(db << "SELECT nodeName FROM claim WHERE nodeName BETWEEN ?1 AND ?2 "
"AND nodeName != ?2 AND activationHeight < ?3 AND expirationHeight > ?3 ORDER BY nodeName LIMIT 1"
<< nodeName << end << nNextHeight) :
(db << "SELECT nodeName FROM claim INDEXED BY claim_nodeName WHERE SUBSTR(nodeName, 1, ?3) = ?1 "
"AND activationHeight < ?2 AND expirationHeight > ?2 ORDER BY nodeName LIMIT 1"
<< nodeName << nNextHeight << nodeName.size()) :
(db << "SELECT nodeName FROM claim WHERE nodeName LIKE ?1 "
"AND activationHeight < ?2 AND expirationHeight > ?2 ORDER BY nodeName LIMIT 1"
<< nodeName + '%' << nNextHeight);
<< nodeName << nNextHeight << nodeName.size());
for (auto&& row: query) {
std::string shortestMatch;
row >> shortestMatch;