Make claimtrie dynamic link library

Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>

src/Makefile.am

@@ -23,6 +23,7 @@ BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(ICU_CPPFLAGS) $(BOOST_CPPFLAGS)
 
 BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include
 BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS)
+BITCOIN_INCLUDES += -I$(srcdir)/claimtrie
 
 LIBBITCOIN_SERVER=libbitcoin_server.a
 LIBBITCOIN_COMMON=libbitcoin_common.a
@@ -419,7 +420,7 @@ claimtrie_libclaimtrie_a_SOURCES = \
   claimtrie/sqlite/sqlite3.c \
   claimtrie/data.cpp \
   claimtrie/forks.cpp \
-  claimtrie/hash.cpp \
+  claimtrie/hashes.cpp \
   claimtrie/log.cpp \
   claimtrie/trie.cpp \
   claimtrie/txoutpoint.cpp \
@@ -545,7 +546,7 @@ endif
 
 libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS)
 libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1)
-libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL
+libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/claimtrie -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL
 libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 endif

src/claimtrie/CMakeLists.txt

@@ -0,0 +1,176 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(claimtrie)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+include(../../contrib/cmake/cmake/CPM.cmake)
+include(ExternalProject)
+
+set(CLAIMTRIE_SRC
+    data.cpp
+    forks.cpp
+    hashes.cpp
+    log.cpp
+    prefixtrie.cpp
+    trie.cpp
+    txoutpoint.cpp
+    uints.cpp
+)
+
+add_library(claimtrie SHARED ${CLAIMTRIE_SRC})
+
+target_include_directories(claimtrie PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+
+# TODO remove leveldb depends
+set(LEVELDB_SRC
+    ../leveldb/db/builder.cc
+    ../leveldb/db/c.cc
+    ../leveldb/db/dbformat.cc
+    ../leveldb/db/db_impl.cc
+    ../leveldb/db/db_iter.cc
+    ../leveldb/db/dumpfile.cc
+    ../leveldb/db/filename.cc
+    ../leveldb/db/log_reader.cc
+    ../leveldb/db/log_writer.cc
+    ../leveldb/db/memtable.cc
+    ../leveldb/db/repair.cc
+    ../leveldb/db/table_cache.cc
+    ../leveldb/db/version_edit.cc
+    ../leveldb/db/version_set.cc
+    ../leveldb/db/write_batch.cc
+    ../leveldb/table/block_builder.cc
+    ../leveldb/table/block.cc
+    ../leveldb/table/filter_block.cc
+    ../leveldb/table/format.cc
+    ../leveldb/table/iterator.cc
+    ../leveldb/table/merger.cc
+    ../leveldb/table/table_builder.cc
+    ../leveldb/table/table.cc
+    ../leveldb/table/two_level_iterator.cc
+    ../leveldb/util/arena.cc
+    ../leveldb/util/bloom.cc
+    ../leveldb/util/cache.cc
+    ../leveldb/util/coding.cc
+    ../leveldb/util/comparator.cc
+    ../leveldb/util/crc32c.cc
+    ../leveldb/util/env.cc
+    ../leveldb/util/env_posix.cc
+    ../leveldb/util/filter_policy.cc
+    ../leveldb/util/hash.cc
+    ../leveldb/util/histogram.cc
+    ../leveldb/util/logging.cc
+    ../leveldb/util/options.cc
+    ../leveldb/util/status.cc
+)
+
+if(WIN32)
+    set(LEVELDB_SRC ${LEVELDB_SRC}
+        ../leveldb/util/env_win.cc
+        ../leveldb/port/port_win.cc
+    )
+else()
+    set(LEVELDB_SRC ${LEVELDB_SRC}
+        ../leveldb/port/port_posix.cc
+    )
+endif()
+
+add_library(leveldb STATIC ${LEVELDB_SRC})
+
+target_include_directories(leveldb PRIVATE ../leveldb ../leveldb/include)
+
+if(WIN32)
+    target_compile_definitions(leveldb PRIVATE -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1)
+else()
+    target_compile_definitions(leveldb PRIVATE -DLEVELDB_PLATFORM_POSIX)
+endif()
+
+target_compile_definitions(leveldb PRIVATE -DLEVELDB_ATOMIC_PRESENT -D__STDC_LIMIT_MACROS)
+
+target_link_libraries(claimtrie PRIVATE leveldb)
+# leveldb depends end here
+
+# TODO remove BITCOIN depends
+target_include_directories(claimtrie PRIVATE ../ ../leveldb/include)
+# it's needed a configure call to create config files
+target_compile_definitions(claimtrie PRIVATE -DHAVE_CONFIG_H)
+# bitcoin depends end here
+
+CPMAddPackage(
+    NAME OpenSSL
+    GITHUB_REPOSITORY openssl/openssl
+    VERSION 1.0.2
+    GIT_TAG OpenSSL_1_0_2r
+    DOWNLOAD_ONLY TRUE
+)
+
+if(OpenSSL_ADDED)
+    string(TOLOWER ${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR} ARCH)
+    ExternalProject_Add(OpenSSL
+        PREFIX openssl
+        SOURCE_DIR ${OpenSSL_SOURCE_DIR}
+        CONFIGURE_COMMAND ${OpenSSL_SOURCE_DIR}/Configure ${ARCH} no-shared no-dso no-engines -fPIC --prefix=<INSTALL_DIR>
+        BUILD_IN_SOURCE 1
+    )
+    add_dependencies(claimtrie OpenSSL)
+    ExternalProject_Get_Property(OpenSSL INSTALL_DIR)
+    target_link_directories(claimtrie PRIVATE ${INSTALL_DIR}/lib)
+    target_include_directories(claimtrie PRIVATE ${INSTALL_DIR}/include)
+endif(OpenSSL_ADDED)
+
+target_link_libraries(claimtrie PRIVATE ssl)
+
+set(BOOST_LIBS filesystem,locale,system,chrono,thread,test)
+
+set(BOOST_COMPONENTS filesystem;locale;system,chrono,thread,unit_test_framework)
+
+CPMAddPackage(
+    NAME Boost
+    GITHUB_REPOSITORY boostorg/boost
+    VERSION 1.64.0
+    COMPONENTS ${BOOST_COMPONENTS}
+    GIT_TAG boost-1.69.0
+    GIT_SUBMODULES libs/* tools/*
+    DOWNLOAD_ONLY TRUE
+)
+
+# if boost is found system wide we expect to be compiled against icu, so we can skip it
+if(Boost_ADDED)
+    CPMAddPackage(
+        NAME ICU
+        GITHUB_REPOSITORY unicode-org/icu
+        VERSION 63.2
+        GIT_TAG release-63-2
+        DOWNLOAD_ONLY TRUE
+    )
+
+    if(ICU_ADDED)
+        ExternalProject_Add(ICU
+            PREFIX icu
+            SOURCE_DIR ${ICU_SOURCE_DIR}
+            CONFIGURE_COMMAND ${ICU_SOURCE_DIR}/icu4c/source/configure --disable-extras --disable-strict --enable-static
+                --disable-shared --disable-tests --disable-samples --disable-dyload --disable-layoutex CFLAGS=-fPIC CPPFLAGS=-fPIC --prefix=<INSTALL_DIR>
+        )
+        ExternalProject_Get_Property(ICU INSTALL_DIR)
+        set(ICU_PATH ${INSTALL_DIR})
+        target_link_directories(claimtrie PRIVATE ${ICU_PATH}/lib)
+        target_include_directories(claimtrie PRIVATE ${ICU_PATH}/include)
+    endif(ICU_ADDED)
+
+    ExternalProject_Add(Boost
+        PREFIX boost
+        DEPENDS ICU
+        SOURCE_DIR ${Boost_SOURCE_DIR}
+        CONFIGURE_COMMAND ${Boost_SOURCE_DIR}/bootstrap.sh --with-icu=${ICU_PATH} --with-libraries=${BOOST_LIBS} && ${Boost_SOURCE_DIR}/b2 headers
+        BUILD_COMMAND ${Boost_SOURCE_DIR}/b2 install threading=multi -sNO_BZIP2=1 -sNO_ZLIB=1 link=static linkflags="-L${ICU_PATH}/lib -licuio -licuuc -licudata -licui18n" cxxflags=-fPIC boost.locale.iconv=off boost.locale.posix=off boost.locale.icu=on boost.locale.std=off -sICU_PATH=${ICU_PATH} --prefix=<INSTALL_DIR>
+        INSTALL_COMMAND ""
+        BUILD_IN_SOURCE 1
+    )
+    add_dependencies(claimtrie Boost)
+    ExternalProject_Get_Property(Boost INSTALL_DIR)
+    target_link_directories(claimtrie PRIVATE ${INSTALL_DIR}/lib)
+    target_include_directories(claimtrie PRIVATE ${INSTALL_DIR}/include)
+    set_property(DIRECTORY PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${Boost_SOURCE_DIR}/bin.v2)
+endif(Boost_ADDED)
+
+target_link_libraries(claimtrie PRIVATE boost_filesystem boost_locale)

src/claimtrie/data.cpp

@@ -1,6 +1,6 @@
 
-#include <claimtrie/data.h>
-#include <claimtrie/log.h>
+#include <data.h>
+#include <log.h>
 
 #include <algorithm>
 #include <sstream>

src/claimtrie/data.h

@@ -3,8 +3,8 @@
 #define CLAIMTRIE_DATA_H
 
 #include <claimtrie/sqlite/sqlite3.h>
-#include <claimtrie/txoutpoint.h>
-#include <claimtrie/uints.h>
+#include <txoutpoint.h>
+#include <uints.h>
 
 #include <cstring>
 #include <string>

src/claimtrie/forks.cpp

@@ -1,8 +1,8 @@
 
-#include <claimtrie/forks.h>
-#include <claimtrie/hash.h>
-#include <claimtrie/log.h>
-#include <claimtrie/trie.h>
+#include <forks.h>
+#include <hashes.h>
+#include <log.h>
+#include <trie.h>
 
 #include <boost/locale.hpp>
 #include <boost/locale/conversion.hpp>

src/claimtrie/forks.h

@@ -2,7 +2,7 @@
 #ifndef CLAIMTRIE_FORKS_H
 #define CLAIMTRIE_FORKS_H
 
-#include <claimtrie/trie.h>
+#include <trie.h>
 
 class CClaimTrieCacheExpirationFork : public CClaimTrieCacheBase
 {

src/claimtrie/hashes.cpp

@@ -1,6 +1,7 @@
 
-#include <claimtrie/hash.h>
+#include <hashes.h>
 
+// Bitcoin doubles hash
 CUint256 CalcHash(SHA256_CTX* sha)
 {
     CUint256 result;

src/claimtrie/hashes.h

@@ -4,9 +4,8 @@
 
 #include <openssl/sha.h>
 
-#include <claimtrie/uints.h>
+#include <uints.h>
 
-// Bitcoin doubles hashes
 CUint256 CalcHash(SHA256_CTX* sha);
 
 template<typename TIterator, typename... Args>

src/claimtrie/log.cpp

@@ -1,5 +1,5 @@
 
-#include <claimtrie/log.h>
+#include <log.h>
 
 void CLogPrint::setLogger(ClogBase* log)
 {

src/claimtrie/trie.cpp

@@ -1,11 +1,13 @@
 
-#include <claimtrie/hash.h>
-#include <claimtrie/log.h>
-#include <claimtrie/trie.h>
+#include <forks.h>
+#include <hashes.h>
+#include <log.h>
+#include <trie.h>
 
 #include <algorithm>
 #include <memory>
 #include <iomanip>
+#include <tuple>
 
 #include <boost/container/flat_map.hpp>
 #include <boost/container/flat_set.hpp>
@@ -446,23 +448,21 @@ CUint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(const std::string& name
     std::vector<uint8_t> vchToHash;
     const auto pos = name.size();
     // we have to free up the hash query so it can be reused by a child
-    struct Triple { std::string name; std::unique_ptr<CUint256> hash; int takeoverHeight; };
-    std::vector<Triple> children;
-    for (auto&& row : childHashQuery << name) {
-        children.emplace_back();
-        auto& b = children.back();
-        row >> b.name >> b.hash >> b.takeoverHeight;
+    std::vector<std::tuple<std::string, std::unique_ptr<CUint256>, int>> children;
+    childHashQuery << name >> [&children](std::string name, std::unique_ptr<CUint256> hash, int takeoverHeight) {
+        children.push_back(std::make_tuple(std::move(name), std::move(hash), takeoverHeight));
     }
     childHashQuery++;
 
     for (auto& child: children) {
-        if (child.hash == nullptr) child.hash = std::make_unique<CUint256>();
-        if (child.hash->IsNull()) {
-            *child.hash = recursiveComputeMerkleHash(child.name, child.takeoverHeight, checkOnly);
-        }
-        completeHash(*child.hash, child.name, pos);
-        vchToHash.push_back(child.name[pos]);
-        vchToHash.insert(vchToHash.end(), child.hash->begin(), child.hash->end());
+        auto& name = std::get<0>(child);
+        auto& hash = std::get<1>(child);
+        if (!hash) hash = std::make_unique<CUint256>();
+        if (hash->IsNull())
+            hash = recursiveComputeMerkleHash(name, std::get<2>(child), checkOnly);
+        completeHash(*hash, name, pos);
+        vchToHash.push_back(name[pos]);
+        vchToHash.insert(vchToHash.end(), hash->begin(), hash->end());
     }
 
     CClaimValue claim;
@@ -494,6 +494,21 @@ bool CClaimTrieCacheBase::checkConsistency()
     return true;
 }
 
+bool CClaimTrieCacheBase::validateDb(const CUint256& rootHash)
+{
+    logPrint << "Checking claim trie consistency... ";
+    if (checkConsistency()) {
+        logPrint << "consistent" << Clog::endl;
+        if (rootHash != getMerkleHash()) {
+            logPrint << "CClaimTrieCacheBase::" << __func__ << "(): the block's root claim hash doesn't match the persisted claim root hash." << Clog::endl;
+            return false;
+        }
+        return true;
+    }
+    logPrint << "inconsistent!" << Clog::endl;
+    return false;
+}
+
 bool CClaimTrieCacheBase::flush()
 {
     if (transacting) {
@@ -519,23 +534,6 @@ bool CClaimTrieCacheBase::flush()
     return true;
 }
 
-bool CClaimTrieCacheBase::ReadFromDisk(int nHeight, const CUint256& rootHash)
-{
-    base->nNextHeight = nNextHeight = nHeight + 1;
-
-    logPrint << "Checking claim trie consistency... " << Clog::endl;
-    if (checkConsistency()) {
-        logPrint << "consistent" << Clog::endl;
-        if (rootHash != getMerkleHash()) {
-            logPrint << "Merkle hash does not match root hash" << Clog::endl;
-            return false;
-        }
-        return true;
-    }
-    logPrint << "inconsistent!" << Clog::endl;
-    return false;
-}
-
 CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base)
     : base(base), db(base->db), transacting(false),
       childHashQuery(db << "SELECT name, hash, IFNULL(takeoverHeight, 0) FROM nodes WHERE parent = ? ORDER BY name"),

src/claimtrie/trie.h

@@ -1,9 +1,9 @@
 #ifndef CLAIMTRIE_TRIE_H
 #define CLAIMTRIE_TRIE_H
 
-#include <claimtrie/data.h>
-#include <claimtrie/txoutpoint.h>
-#include <claimtrie/uints.h>
+#include <data.h>
+#include <txoutpoint.h>
+#include <uints.h>
 
 #include <functional>
 #include <map>
@@ -133,7 +133,7 @@ public:
     bool flush();
     bool empty() const;
     bool checkConsistency();
-    bool ReadFromDisk(int nHeight, const CUint256& rootHash);
+    bool validateDb(const CUint256& rootHash);
 
     std::size_t getTotalNamesInTrie() const;
     std::size_t getTotalClaimsInTrie() const;
@@ -197,6 +197,7 @@ protected:
     void ensureTreeStructureIsUpToDate();
 
 private:
+    std::unordered_map<std::string, std::pair<CUint160, int>> takeoverCache;
     // for unit test
     friend struct ClaimTrieChainFixture;
     friend class CClaimTrieCacheTest;

src/claimtrie/txoutpoint.cpp

@@ -1,5 +1,5 @@
 
-#include <claimtrie/txoutpoint.h>
+#include <txoutpoint.h>
 
 #include <sstream>
 

src/claimtrie/txoutpoint.h

@@ -2,7 +2,7 @@
 #ifndef CLAIMTRIE_TXOUTPUT_H
 #define CLAIMTRIE_TXOUTPUT_H
 
-#include <claimtrie/uints.h>
+#include <uints.h>
 
 #include <algorithm>
 #include <type_traits>

src/claimtrie/uints.cpp

@@ -1,5 +1,5 @@
 
-#include <claimtrie/uints.h>
+#include <uints.h>
 
 #include <algorithm>
 #include <cassert>

src/init.cpp

@@ -1539,8 +1539,7 @@ bool AppInitMain()
                 }
                 auto tip = chainActive.Tip();
                 assert(tip);
-                CClaimTrieCache trieCache(pclaimTrie);
-                if (!trieCache.ReadFromDisk(tip->nHeight, tip->hashClaimTrie)) {
+                if (!CClaimTrieCache(pclaimTrie).validateDb(tip->hashClaimTrie))
                     strLoadError = _("Error loading the claim trie from disk");
                     break;
                 }

src/test/claimtrieexpirationfork_tests.cpp

@@ -277,7 +277,7 @@ BOOST_AUTO_TEST_CASE(hardfork_disk_test)
     fixture.IncrementBlocks(7, true);
     BOOST_CHECK_EQUAL(fixture.expirationTime(), 6);
     auto tip = chainActive.Tip();
-    fixture.ReadFromDisk(tip->nHeight, tip->hashClaimTrie);
+    pclaimTrie->ReadFromDisk(tip->nHeight, tip->hashClaimTrie);
     BOOST_CHECK_EQUAL(fixture.expirationTime(), 6);
 
     // Create a claim and support 1 block before the fork height that will expire after the fork height.
@@ -289,7 +289,7 @@ BOOST_AUTO_TEST_CASE(hardfork_disk_test)
     fixture.IncrementBlocks(1);
 
     tip = chainActive.Tip();
-    fixture.ReadFromDisk(tip->nHeight, tip->hashClaimTrie);
+    pclaimTrie->ReadFromDisk(tip->nHeight, tip->hashClaimTrie);
     BOOST_CHECK_EQUAL(fixture.expirationTime(), 3);
     fixture.IncrementBlocks(1);
     BOOST_CHECK_EQUAL(fixture.expirationTime(), 6);
@@ -311,7 +311,7 @@ BOOST_AUTO_TEST_CASE(hardfork_disk_test)
     CMutableTransaction s2 = fixture.MakeSupport(fixture.GetCoinbase(),tx2,"test2",1);
     fixture.IncrementBlocks(1);
     tip = chainActive.Tip();
-    fixture.ReadFromDisk(tip->nHeight, tip->hashClaimTrie);
+    pclaimTrie->ReadFromDisk(tip->nHeight, tip->hashClaimTrie);
     CMutableTransaction u2 = fixture.MakeUpdate(tx2, "test2", "two", ClaimIdHash(tx2.GetHash(), 0), 1);
     // increment to fork
     fixture.IncrementBlocks(2);