From c6334a5ef2a05391074b1315c7c6cf64f5f25b9e Mon Sep 17 00:00:00 2001 From: Anthony Fieroni Date: Tue, 22 Oct 2019 10:57:05 +0300 Subject: [PATCH] Make claimtrie dynamic link library Signed-off-by: Anthony Fieroni --- src/Makefile.am | 5 +- src/claimtrie/CMakeLists.txt | 176 +++++++++++++++++++++ src/claimtrie/data.cpp | 4 +- src/claimtrie/data.h | 4 +- src/claimtrie/forks.cpp | 8 +- src/claimtrie/forks.h | 2 +- src/claimtrie/{hash.cpp => hashes.cpp} | 3 +- src/claimtrie/{hash.h => hashes.h} | 3 +- src/claimtrie/log.cpp | 2 +- src/claimtrie/trie.cpp | 64 ++++---- src/claimtrie/trie.h | 9 +- src/claimtrie/txoutpoint.cpp | 2 +- src/claimtrie/txoutpoint.h | 2 +- src/claimtrie/uints.cpp | 2 +- src/init.cpp | 3 +- src/test/claimtrieexpirationfork_tests.cpp | 6 +- 16 files changed, 235 insertions(+), 60 deletions(-) create mode 100644 src/claimtrie/CMakeLists.txt rename src/claimtrie/{hash.cpp => hashes.cpp} (84%) rename src/claimtrie/{hash.h => hashes.h} (93%) diff --git a/src/Makefile.am b/src/Makefile.am index d81340626..85165466e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -23,6 +23,7 @@ BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(ICU_CPPFLAGS) $(BOOST_CPPFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) +BITCOIN_INCLUDES += -I$(srcdir)/claimtrie LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -419,7 +420,7 @@ claimtrie_libclaimtrie_a_SOURCES = \ claimtrie/sqlite/sqlite3.c \ claimtrie/data.cpp \ claimtrie/forks.cpp \ - claimtrie/hash.cpp \ + claimtrie/hashes.cpp \ claimtrie/log.cpp \ claimtrie/trie.cpp \ claimtrie/txoutpoint.cpp \ @@ -545,7 +546,7 @@ endif libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) -libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL +libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/claimtrie -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) endif diff --git a/src/claimtrie/CMakeLists.txt b/src/claimtrie/CMakeLists.txt new file mode 100644 index 000000000..4563b53df --- /dev/null +++ b/src/claimtrie/CMakeLists.txt @@ -0,0 +1,176 @@ +cmake_minimum_required(VERSION 3.10) + +project(claimtrie) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +include(../../contrib/cmake/cmake/CPM.cmake) +include(ExternalProject) + +set(CLAIMTRIE_SRC + data.cpp + forks.cpp + hashes.cpp + log.cpp + prefixtrie.cpp + trie.cpp + txoutpoint.cpp + uints.cpp +) + +add_library(claimtrie SHARED ${CLAIMTRIE_SRC}) + +target_include_directories(claimtrie PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + +# TODO remove leveldb depends +set(LEVELDB_SRC + ../leveldb/db/builder.cc + ../leveldb/db/c.cc + ../leveldb/db/dbformat.cc + ../leveldb/db/db_impl.cc + ../leveldb/db/db_iter.cc + ../leveldb/db/dumpfile.cc + ../leveldb/db/filename.cc + ../leveldb/db/log_reader.cc + ../leveldb/db/log_writer.cc + ../leveldb/db/memtable.cc + ../leveldb/db/repair.cc + ../leveldb/db/table_cache.cc + ../leveldb/db/version_edit.cc + ../leveldb/db/version_set.cc + ../leveldb/db/write_batch.cc + ../leveldb/table/block_builder.cc + ../leveldb/table/block.cc + ../leveldb/table/filter_block.cc + ../leveldb/table/format.cc + ../leveldb/table/iterator.cc + ../leveldb/table/merger.cc + ../leveldb/table/table_builder.cc + ../leveldb/table/table.cc + ../leveldb/table/two_level_iterator.cc + ../leveldb/util/arena.cc + ../leveldb/util/bloom.cc + ../leveldb/util/cache.cc + ../leveldb/util/coding.cc + ../leveldb/util/comparator.cc + ../leveldb/util/crc32c.cc + ../leveldb/util/env.cc + ../leveldb/util/env_posix.cc + ../leveldb/util/filter_policy.cc + ../leveldb/util/hash.cc + ../leveldb/util/histogram.cc + ../leveldb/util/logging.cc + ../leveldb/util/options.cc + ../leveldb/util/status.cc +) + +if(WIN32) + set(LEVELDB_SRC ${LEVELDB_SRC} + ../leveldb/util/env_win.cc + ../leveldb/port/port_win.cc + ) +else() + set(LEVELDB_SRC ${LEVELDB_SRC} + ../leveldb/port/port_posix.cc + ) +endif() + +add_library(leveldb STATIC ${LEVELDB_SRC}) + +target_include_directories(leveldb PRIVATE ../leveldb ../leveldb/include) + +if(WIN32) + target_compile_definitions(leveldb PRIVATE -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1) +else() + target_compile_definitions(leveldb PRIVATE -DLEVELDB_PLATFORM_POSIX) +endif() + +target_compile_definitions(leveldb PRIVATE -DLEVELDB_ATOMIC_PRESENT -D__STDC_LIMIT_MACROS) + +target_link_libraries(claimtrie PRIVATE leveldb) +# leveldb depends end here + +# TODO remove BITCOIN depends +target_include_directories(claimtrie PRIVATE ../ ../leveldb/include) +# it's needed a configure call to create config files +target_compile_definitions(claimtrie PRIVATE -DHAVE_CONFIG_H) +# bitcoin depends end here + +CPMAddPackage( + NAME OpenSSL + GITHUB_REPOSITORY openssl/openssl + VERSION 1.0.2 + GIT_TAG OpenSSL_1_0_2r + DOWNLOAD_ONLY TRUE +) + +if(OpenSSL_ADDED) + string(TOLOWER ${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR} ARCH) + ExternalProject_Add(OpenSSL + PREFIX openssl + SOURCE_DIR ${OpenSSL_SOURCE_DIR} + CONFIGURE_COMMAND ${OpenSSL_SOURCE_DIR}/Configure ${ARCH} no-shared no-dso no-engines -fPIC --prefix= + BUILD_IN_SOURCE 1 + ) + add_dependencies(claimtrie OpenSSL) + ExternalProject_Get_Property(OpenSSL INSTALL_DIR) + target_link_directories(claimtrie PRIVATE ${INSTALL_DIR}/lib) + target_include_directories(claimtrie PRIVATE ${INSTALL_DIR}/include) +endif(OpenSSL_ADDED) + +target_link_libraries(claimtrie PRIVATE ssl) + +set(BOOST_LIBS filesystem,locale,system,chrono,thread,test) + +set(BOOST_COMPONENTS filesystem;locale;system,chrono,thread,unit_test_framework) + +CPMAddPackage( + NAME Boost + GITHUB_REPOSITORY boostorg/boost + VERSION 1.64.0 + COMPONENTS ${BOOST_COMPONENTS} + GIT_TAG boost-1.69.0 + GIT_SUBMODULES libs/* tools/* + DOWNLOAD_ONLY TRUE +) + +# if boost is found system wide we expect to be compiled against icu, so we can skip it +if(Boost_ADDED) + CPMAddPackage( + NAME ICU + GITHUB_REPOSITORY unicode-org/icu + VERSION 63.2 + GIT_TAG release-63-2 + DOWNLOAD_ONLY TRUE + ) + + if(ICU_ADDED) + ExternalProject_Add(ICU + PREFIX icu + SOURCE_DIR ${ICU_SOURCE_DIR} + CONFIGURE_COMMAND ${ICU_SOURCE_DIR}/icu4c/source/configure --disable-extras --disable-strict --enable-static + --disable-shared --disable-tests --disable-samples --disable-dyload --disable-layoutex CFLAGS=-fPIC CPPFLAGS=-fPIC --prefix= + ) + ExternalProject_Get_Property(ICU INSTALL_DIR) + set(ICU_PATH ${INSTALL_DIR}) + target_link_directories(claimtrie PRIVATE ${ICU_PATH}/lib) + target_include_directories(claimtrie PRIVATE ${ICU_PATH}/include) + endif(ICU_ADDED) + + ExternalProject_Add(Boost + PREFIX boost + DEPENDS ICU + SOURCE_DIR ${Boost_SOURCE_DIR} + CONFIGURE_COMMAND ${Boost_SOURCE_DIR}/bootstrap.sh --with-icu=${ICU_PATH} --with-libraries=${BOOST_LIBS} && ${Boost_SOURCE_DIR}/b2 headers + BUILD_COMMAND ${Boost_SOURCE_DIR}/b2 install threading=multi -sNO_BZIP2=1 -sNO_ZLIB=1 link=static linkflags="-L${ICU_PATH}/lib -licuio -licuuc -licudata -licui18n" cxxflags=-fPIC boost.locale.iconv=off boost.locale.posix=off boost.locale.icu=on boost.locale.std=off -sICU_PATH=${ICU_PATH} --prefix= + INSTALL_COMMAND "" + BUILD_IN_SOURCE 1 + ) + add_dependencies(claimtrie Boost) + ExternalProject_Get_Property(Boost INSTALL_DIR) + target_link_directories(claimtrie PRIVATE ${INSTALL_DIR}/lib) + target_include_directories(claimtrie PRIVATE ${INSTALL_DIR}/include) + set_property(DIRECTORY PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${Boost_SOURCE_DIR}/bin.v2) +endif(Boost_ADDED) + +target_link_libraries(claimtrie PRIVATE boost_filesystem boost_locale) diff --git a/src/claimtrie/data.cpp b/src/claimtrie/data.cpp index bf0de4fc8..77a4ba70e 100644 --- a/src/claimtrie/data.cpp +++ b/src/claimtrie/data.cpp @@ -1,6 +1,6 @@ -#include -#include +#include +#include #include #include diff --git a/src/claimtrie/data.h b/src/claimtrie/data.h index aa1bb2e88..b1801521a 100644 --- a/src/claimtrie/data.h +++ b/src/claimtrie/data.h @@ -3,8 +3,8 @@ #define CLAIMTRIE_DATA_H #include -#include -#include +#include +#include #include #include diff --git a/src/claimtrie/forks.cpp b/src/claimtrie/forks.cpp index 19b0fc94a..67b40ff9e 100644 --- a/src/claimtrie/forks.cpp +++ b/src/claimtrie/forks.cpp @@ -1,8 +1,8 @@ -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/src/claimtrie/forks.h b/src/claimtrie/forks.h index 46573e219..f88765cca 100644 --- a/src/claimtrie/forks.h +++ b/src/claimtrie/forks.h @@ -2,7 +2,7 @@ #ifndef CLAIMTRIE_FORKS_H #define CLAIMTRIE_FORKS_H -#include +#include class CClaimTrieCacheExpirationFork : public CClaimTrieCacheBase { diff --git a/src/claimtrie/hash.cpp b/src/claimtrie/hashes.cpp similarity index 84% rename from src/claimtrie/hash.cpp rename to src/claimtrie/hashes.cpp index 8d18ecbb1..1debebdbe 100644 --- a/src/claimtrie/hash.cpp +++ b/src/claimtrie/hashes.cpp @@ -1,6 +1,7 @@ -#include +#include +// Bitcoin doubles hash CUint256 CalcHash(SHA256_CTX* sha) { CUint256 result; diff --git a/src/claimtrie/hash.h b/src/claimtrie/hashes.h similarity index 93% rename from src/claimtrie/hash.h rename to src/claimtrie/hashes.h index 88127c4c3..f03db1487 100644 --- a/src/claimtrie/hash.h +++ b/src/claimtrie/hashes.h @@ -4,9 +4,8 @@ #include -#include +#include -// Bitcoin doubles hashes CUint256 CalcHash(SHA256_CTX* sha); template diff --git a/src/claimtrie/log.cpp b/src/claimtrie/log.cpp index 364cde754..a17a97938 100644 --- a/src/claimtrie/log.cpp +++ b/src/claimtrie/log.cpp @@ -1,5 +1,5 @@ -#include +#include void CLogPrint::setLogger(ClogBase* log) { diff --git a/src/claimtrie/trie.cpp b/src/claimtrie/trie.cpp index 452f266cd..6f4fb3932 100644 --- a/src/claimtrie/trie.cpp +++ b/src/claimtrie/trie.cpp @@ -1,11 +1,13 @@ -#include -#include -#include +#include +#include +#include +#include #include #include #include +#include #include #include @@ -446,23 +448,21 @@ CUint256 CClaimTrieCacheBase::recursiveComputeMerkleHash(const std::string& name std::vector vchToHash; const auto pos = name.size(); // we have to free up the hash query so it can be reused by a child - struct Triple { std::string name; std::unique_ptr hash; int takeoverHeight; }; - std::vector children; - for (auto&& row : childHashQuery << name) { - children.emplace_back(); - auto& b = children.back(); - row >> b.name >> b.hash >> b.takeoverHeight; + std::vector, int>> children; + childHashQuery << name >> [&children](std::string name, std::unique_ptr hash, int takeoverHeight) { + children.push_back(std::make_tuple(std::move(name), std::move(hash), takeoverHeight)); } childHashQuery++; for (auto& child: children) { - if (child.hash == nullptr) child.hash = std::make_unique(); - if (child.hash->IsNull()) { - *child.hash = recursiveComputeMerkleHash(child.name, child.takeoverHeight, checkOnly); - } - completeHash(*child.hash, child.name, pos); - vchToHash.push_back(child.name[pos]); - vchToHash.insert(vchToHash.end(), child.hash->begin(), child.hash->end()); + auto& name = std::get<0>(child); + auto& hash = std::get<1>(child); + if (!hash) hash = std::make_unique(); + if (hash->IsNull()) + hash = recursiveComputeMerkleHash(name, std::get<2>(child), checkOnly); + completeHash(*hash, name, pos); + vchToHash.push_back(name[pos]); + vchToHash.insert(vchToHash.end(), hash->begin(), hash->end()); } CClaimValue claim; @@ -494,6 +494,21 @@ bool CClaimTrieCacheBase::checkConsistency() return true; } +bool CClaimTrieCacheBase::validateDb(const CUint256& rootHash) +{ + logPrint << "Checking claim trie consistency... "; + if (checkConsistency()) { + logPrint << "consistent" << Clog::endl; + if (rootHash != getMerkleHash()) { + logPrint << "CClaimTrieCacheBase::" << __func__ << "(): the block's root claim hash doesn't match the persisted claim root hash." << Clog::endl; + return false; + } + return true; + } + logPrint << "inconsistent!" << Clog::endl; + return false; +} + bool CClaimTrieCacheBase::flush() { if (transacting) { @@ -519,23 +534,6 @@ bool CClaimTrieCacheBase::flush() return true; } -bool CClaimTrieCacheBase::ReadFromDisk(int nHeight, const CUint256& rootHash) -{ - base->nNextHeight = nNextHeight = nHeight + 1; - - logPrint << "Checking claim trie consistency... " << Clog::endl; - if (checkConsistency()) { - logPrint << "consistent" << Clog::endl; - if (rootHash != getMerkleHash()) { - logPrint << "Merkle hash does not match root hash" << Clog::endl; - return false; - } - return true; - } - logPrint << "inconsistent!" << Clog::endl; - return false; -} - CClaimTrieCacheBase::CClaimTrieCacheBase(CClaimTrie* base) : base(base), db(base->db), transacting(false), childHashQuery(db << "SELECT name, hash, IFNULL(takeoverHeight, 0) FROM nodes WHERE parent = ? ORDER BY name"), diff --git a/src/claimtrie/trie.h b/src/claimtrie/trie.h index 31e4c73c6..9c7dbaa9c 100644 --- a/src/claimtrie/trie.h +++ b/src/claimtrie/trie.h @@ -1,9 +1,9 @@ #ifndef CLAIMTRIE_TRIE_H #define CLAIMTRIE_TRIE_H -#include -#include -#include +#include +#include +#include #include #include @@ -133,7 +133,7 @@ public: bool flush(); bool empty() const; bool checkConsistency(); - bool ReadFromDisk(int nHeight, const CUint256& rootHash); + bool validateDb(const CUint256& rootHash); std::size_t getTotalNamesInTrie() const; std::size_t getTotalClaimsInTrie() const; @@ -197,6 +197,7 @@ protected: void ensureTreeStructureIsUpToDate(); private: + std::unordered_map> takeoverCache; // for unit test friend struct ClaimTrieChainFixture; friend class CClaimTrieCacheTest; diff --git a/src/claimtrie/txoutpoint.cpp b/src/claimtrie/txoutpoint.cpp index 20c4c550d..ec4e3618f 100644 --- a/src/claimtrie/txoutpoint.cpp +++ b/src/claimtrie/txoutpoint.cpp @@ -1,5 +1,5 @@ -#include +#include #include diff --git a/src/claimtrie/txoutpoint.h b/src/claimtrie/txoutpoint.h index d057e69bf..1ac118a86 100644 --- a/src/claimtrie/txoutpoint.h +++ b/src/claimtrie/txoutpoint.h @@ -2,7 +2,7 @@ #ifndef CLAIMTRIE_TXOUTPUT_H #define CLAIMTRIE_TXOUTPUT_H -#include +#include #include #include diff --git a/src/claimtrie/uints.cpp b/src/claimtrie/uints.cpp index b0e4cc8e7..5ed4bb2a3 100644 --- a/src/claimtrie/uints.cpp +++ b/src/claimtrie/uints.cpp @@ -1,5 +1,5 @@ -#include +#include #include #include diff --git a/src/init.cpp b/src/init.cpp index d3082efce..75d6b95ad 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1539,8 +1539,7 @@ bool AppInitMain() } auto tip = chainActive.Tip(); assert(tip); - CClaimTrieCache trieCache(pclaimTrie); - if (!trieCache.ReadFromDisk(tip->nHeight, tip->hashClaimTrie)) { + if (!CClaimTrieCache(pclaimTrie).validateDb(tip->hashClaimTrie)) strLoadError = _("Error loading the claim trie from disk"); break; } diff --git a/src/test/claimtrieexpirationfork_tests.cpp b/src/test/claimtrieexpirationfork_tests.cpp index efc981b60..a7d2837ea 100644 --- a/src/test/claimtrieexpirationfork_tests.cpp +++ b/src/test/claimtrieexpirationfork_tests.cpp @@ -277,7 +277,7 @@ BOOST_AUTO_TEST_CASE(hardfork_disk_test) fixture.IncrementBlocks(7, true); BOOST_CHECK_EQUAL(fixture.expirationTime(), 6); auto tip = chainActive.Tip(); - fixture.ReadFromDisk(tip->nHeight, tip->hashClaimTrie); + pclaimTrie->ReadFromDisk(tip->nHeight, tip->hashClaimTrie); BOOST_CHECK_EQUAL(fixture.expirationTime(), 6); // Create a claim and support 1 block before the fork height that will expire after the fork height. @@ -289,7 +289,7 @@ BOOST_AUTO_TEST_CASE(hardfork_disk_test) fixture.IncrementBlocks(1); tip = chainActive.Tip(); - fixture.ReadFromDisk(tip->nHeight, tip->hashClaimTrie); + pclaimTrie->ReadFromDisk(tip->nHeight, tip->hashClaimTrie); BOOST_CHECK_EQUAL(fixture.expirationTime(), 3); fixture.IncrementBlocks(1); BOOST_CHECK_EQUAL(fixture.expirationTime(), 6); @@ -311,7 +311,7 @@ BOOST_AUTO_TEST_CASE(hardfork_disk_test) CMutableTransaction s2 = fixture.MakeSupport(fixture.GetCoinbase(),tx2,"test2",1); fixture.IncrementBlocks(1); tip = chainActive.Tip(); - fixture.ReadFromDisk(tip->nHeight, tip->hashClaimTrie); + pclaimTrie->ReadFromDisk(tip->nHeight, tip->hashClaimTrie); CMutableTransaction u2 = fixture.MakeUpdate(tx2, "test2", "two", ClaimIdHash(tx2.GetHash(), 0), 1); // increment to fork fixture.IncrementBlocks(2);