flush more, disk sync less

minor optimizations compiles but still calls fcntl64 updated to expect backwards symbol table reduce output on syncing testnet changing to gcc-only build for Linux fix DOCKER_IMAGE remove docker_build_image optimizing the wallet to better handle claimname calls post-merge fixes
2020-02-19 16:29:38 -07:00 · 2020-02-19 16:29:38 -07:00 · daf90c3adf
commit daf90c3adf
parent 7f4ddab539
16 changed files with 3940 additions and 136 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -14,7 +14,7 @@ jobs:
    - &build-template
      stage: build
      name: linux
-      env: NAME=linux EXT=
+      env: NAME=linux DOCKER_IMAGE=lbry/build_lbrycrd_gcc EXT=
      os: linux
      dist: xenial
      language: minimal
@ -22,7 +22,7 @@ jobs:
        - docker
      install:
        - mkdir -p ${HOME}/ccache
-        - docker pull $DOCKER_BUILD_IMAGE
+        - docker pull $DOCKER_IMAGE
      script:
        - docker run -v "$(pwd):/lbrycrd" -v "${HOME}/ccache:/ccache" -w /lbrycrd -e CCACHE_DIR=/ccache ${DOCKER_IMAGE} packaging/build_${NAME}_64bit.sh
      before_deploy:
@ -55,11 +55,11 @@ jobs:

    - <<: *build-template
      name: windows
-      env: NAME=windows EXT=.exe
+      env: NAME=windows DOCKER_IMAGE=lbry/build_lbrycrd EXT=.exe

    - <<: *build-template
      name: osx
-      env: NAME=darwin EXT=
+      env: NAME=darwin DOCKER_IMAGE=lbry/build_lbrycrd EXT=
      before_install:
        - mkdir -p ./depends/SDKs && pushd depends/SDKs && curl -C - ${MAC_OS_SDK} | tar --skip-old-files -xJ && popd

@ -85,8 +85,8 @@ jobs:
      services:
        - docker
      script:
-        - docker pull $DOCKER_WINE_IMAGE
-        - docker run -v "$(pwd):/test" -e "WINEDEBUG=-all" -e "TRIEHASH_FUZZER_BLOCKS=1000" -it $DOCKER_WINE_IMAGE wine "/test/test_lbrycrd.exe"
+        - docker pull lbry/wine
+        - docker run -v "$(pwd):/test" -e "WINEDEBUG=-all" -e "TRIEHASH_FUZZER_BLOCKS=1000" -it lbry/wine wine "/test/test_lbrycrd.exe"

    - <<: *test-template
      os: osx
--- a/depends/glibc_version_header/force_link_glibc_2.19.h
+++ b/depends/glibc_version_header/force_link_glibc_2.19.h
--- a/depends/hosts/linux.mk
+++ b/depends/hosts/linux.mk
@ -1,7 +1,7 @@
 linux_CFLAGS=-pipe
 linux_CXXFLAGS=$(linux_CFLAGS) -std=c++11

-linux_release_CFLAGS=-O2 -g
+linux_release_CFLAGS=-O3 -g -include $(BASEDIR)/glibc_version_header/force_link_glibc_2.19.h
 linux_release_CXXFLAGS=$(linux_release_CFLAGS)

 linux_debug_CFLAGS=-O1 -g
--- a/depends/packages/icu.mk
+++ b/depends/packages/icu.mk
@ -29,8 +29,8 @@ define $(package)_config_cmds
  PKG_CONFIG_SYSROOT_DIR=/ \
  PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig \
  PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig \
-  sed -i.old 's/^GEN_DEPS.c=.*/& $($(package)_cflags)/' config/mh-mingw* && \
-  sed -i.old 's/^GEN_DEPS.cc=.*/& $($(package)_cxxflags)/' config/mh-mingw* && \
+  sed -i.old 's|^GEN_DEPS.c=.*|& $($(package)_cflags)|' config/mh-mingw* && \
+  sed -i.old 's|^GEN_DEPS.cc=.*|& $($(package)_cxxflags)|' config/mh-mingw* && \
  $($(package)_autoconf)
 endef

--- a/depends/packages/openssl.mk
+++ b/depends/packages/openssl.mk
@ -6,7 +6,7 @@ $(package)_sha256_hash=8f9faeaebad088e772f4ef5e38252d472be4d878c6b3a2718c10a4fce
 $(package)_patches=0001-Add-OpenSSL-termios-fix-for-musl-libc.patch

 define $(package)_set_vars
-$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
+$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc) $($(package)_cflags) $($(package)_cppflags)"
 $(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
 $(package)_config_opts+=no-camellia
 $(package)_config_opts+=no-capieng
@ -43,7 +43,6 @@ $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-whirlpool
 $(package)_config_opts+=no-zlib
 $(package)_config_opts+=no-zlib-dynamic
-$(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags)
 $(package)_config_opts_linux=-fPIC -Wa,--noexecstack
 $(package)_config_opts_x86_64_linux=linux-x86_64
 $(package)_config_opts_i686_linux=linux-generic32
--- a/packaging/Dockerfile
+++ b/packaging/Dockerfile
@ -1,17 +1,12 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 ENV LANG C.UTF-8

 RUN set -xe; \
    apt-get update; \
    apt-get install --no-install-recommends -y build-essential libtool autotools-dev automake pkg-config git wget apt-utils \
        librsvg2-bin libtiff-tools cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools python3-setuptools xz-utils ccache g++-multilib \
-        g++-mingw-w64-i686 mingw-w64-i686-dev bsdmainutils curl ca-certificates g++-mingw-w64-x86-64 mingw-w64-x86-64-dev; \
-    rm -rf /var/lib/apt/lists/*;
-
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -; \
-    echo 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main' >> /etc/apt/sources.list; \
-    apt-get update; \
-    apt-get install --no-install-recommends -y clang-8 lldb-8 lld-8 libc++-8-dev; \
+        g++-mingw-w64-i686 mingw-w64-i686-dev bsdmainutils curl ca-certificates g++-mingw-w64-x86-64 mingw-w64-x86-64-dev \
+        clang-8 lldb-8 lld-8 libc++-8-dev; \
    rm -rf /var/lib/apt/lists/*;

 RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang-cpp-8 80; \
@ -28,7 +23,7 @@ ARG VCS_REF
 ARG BUILD_DATE
 LABEL maintainer="blockchain@lbry.com" \
    decription="build_lbrycrd" \
-    version="1.1" \
+    version="1.2" \
    org.label-schema.name="build_lbrycrd" \
    org.label-schema.description="Use this to generate a reproducible build of LBRYcrd" \
    org.label-schema.build-date=$BUILD_DATE \
--- a/packaging/docker-for-gcc/Dockerfile
+++ b/packaging/docker-for-gcc/Dockerfile
@ -0,0 +1,28 @@
+FROM ubuntu:18.04
+ENV LANG C.UTF-8
+
+RUN set -xe; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y build-essential libtool autotools-dev automake pkg-config git wget apt-utils \
+        librsvg2-bin cmake libcap-dev libz-dev libbz2-dev python-setuptools python3-setuptools xz-utils ccache \
+        bsdmainutils curl ca-certificates; \
+    rm -rf /var/lib/apt/lists/*; \
+    /usr/sbin/update-ccache-symlinks;
+    
+ARG VCS_REF
+ARG BUILD_DATE
+LABEL maintainer="blockchain@lbry.com" \
+    decription="build_lbrycrd_gcc" \
+    version="1.2" \
+    org.label-schema.name="build_lbrycrd_gcc" \
+    org.label-schema.description="Use this to generate a reproducible build of LBRYcrd for Linux" \
+    org.label-schema.build-date=$BUILD_DATE \
+    org.label-schema.vcs-ref=$VCS_REF \
+    org.label-schema.vcs-url="https://github.com/lbryio/lbrycrd" \
+    org.label-schema.schema-version="1.0.0-rc1" \
+    org.label-schema.vendor="LBRY" \
+    org.label-schema.docker.cmd="docker build --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` -t lbry/build_lbrycrd_gcc packaging"
+
+ENV PATH "/usr/lib/ccache:$PATH"
+WORKDIR /home
+CMD ["/bin/bash"]
--- a/src/init.cpp
+++ b/src/init.cpp
@ -236,7 +236,7 @@ void Shutdown(InitInterfaces& interfaces)
    {
        LOCK(cs_main);
        if (g_chainstate && g_chainstate->CanFlushToDisk()) {
-            g_chainstate->ForceFlushStateToDisk();
+            g_chainstate->ForceFlushStateToDisk(true);
        }
    }

@ -261,7 +261,7 @@ void Shutdown(InitInterfaces& interfaces)
    {
        LOCK(cs_main);
        if (g_chainstate && g_chainstate->CanFlushToDisk()) {
-            g_chainstate->ForceFlushStateToDisk();
+            g_chainstate->ForceFlushStateToDisk(true);
            g_chainstate->ResetCoinsViews();
        }
        pblocktree.reset();
--- a/src/nameclaim.cpp
+++ b/src/nameclaim.cpp
@ -119,10 +119,57 @@ bool DecodeClaimScript(const CScript& scriptIn, int& op, std::vector<std::vector
    vvchParams.push_back(std::move(vchParam2));
    if (last_drop == OP_2DROP)
        vvchParams.push_back(std::move(vchParam3));
-
    return true;
 }

+// this overload exists because optimizer wouldn't inline GetScriptOp:
+bool DecodeClaimScript(const CScript& scriptIn, int& op, CScript::const_iterator& pc, bool allowSupportMetadata)
+{
+    op = -1;
+    opcodetype opcode;
+    if (!scriptIn.GetOp(pc, opcode))
+        return false;
+
+    if (opcode != OP_CLAIM_NAME && opcode != OP_SUPPORT_CLAIM && opcode != OP_UPDATE_CLAIM)
+        return false;
+
+    op = opcode;
+
+    // Valid formats:
+    // OP_CLAIM_NAME vchName vchValue OP_2DROP OP_DROP pubkeyscript
+    // OP_UPDATE_CLAIM vchName vchClaimId vchValue OP_2DROP OP_2DROP pubkeyscript
+    // OP_SUPPORT_CLAIM vchName vchClaimId OP_2DROP OP_DROP pubkeyscript
+    // OP_SUPPORT_CLAIM vchName vchClaimId vchValue OP_2DROP OP_2DROP pubkeyscript
+    // All others are invalid.
+
+    if (!scriptIn.GetOp(pc, opcode) || opcode < 0 || opcode > OP_PUSHDATA4)
+        return false;
+
+    if (!scriptIn.GetOp(pc, opcode) || opcode < 0 || opcode > OP_PUSHDATA4)
+        return false;
+
+    if (!scriptIn.GetOp(pc, opcode))
+        return false;
+
+    auto last_drop = OP_DROP;
+    if (opcode >= 0 && opcode <= OP_PUSHDATA4 && op != OP_CLAIM_NAME) {
+        if (!scriptIn.GetOp(pc, opcode))
+            return false;
+        last_drop = OP_2DROP;
+    } else if (op == OP_UPDATE_CLAIM)
+        return false;
+
+    if (opcode != OP_2DROP)
+        return false;
+
+    if (!scriptIn.GetOp(pc, opcode) || opcode != last_drop)
+        return false;
+
+    return !(op == OP_SUPPORT_CLAIM && last_drop == OP_2DROP && !allowSupportMetadata);
+
+}
+
+
 uint160 ClaimIdHash(const uint256& txhash, uint32_t nOut)
 {
    std::vector<unsigned char> claimToHash(txhash.begin(), txhash.end());
@ -139,10 +186,10 @@ CScript StripClaimScriptPrefix(const CScript& scriptIn)

 CScript StripClaimScriptPrefix(const CScript& scriptIn, int& op)
 {
-    std::vector<std::vector<unsigned char> > vvchParams;
    CScript::const_iterator pc = scriptIn.begin();

-    if (!DecodeClaimScript(scriptIn, op, vvchParams, pc))
+    auto isClaim = DecodeClaimScript(scriptIn, op, pc);
+    if (!isClaim)
        return scriptIn;

    return CScript(pc, scriptIn.end());
--- a/src/nameclaim.h
+++ b/src/nameclaim.h
@ -30,6 +30,7 @@ CScript SupportClaimScript(std::string name, uint160 claimId, std::string value=
 CScript UpdateClaimScript(std::string name, uint160 claimId, std::string value);
 bool DecodeClaimScript(const CScript& scriptIn, int& op, std::vector<std::vector<unsigned char> >& vvchParams, bool allowSupportMetadata=true);
 bool DecodeClaimScript(const CScript& scriptIn, int& op, std::vector<std::vector<unsigned char> >& vvchParams, CScript::const_iterator& pc, bool allowSupportMetadata=true);
+bool DecodeClaimScript(const CScript& scriptIn, int& op, CScript::const_iterator& pc, bool allowSupportMetadata=true);
 CScript StripClaimScriptPrefix(const CScript& scriptIn);
 CScript StripClaimScriptPrefix(const CScript& scriptIn, int& op);
 uint160 ClaimIdHash(const uint256& txhash, uint32_t nOut);
--- a/src/txdb.cpp
+++ b/src/txdb.cpp
@ -94,64 +94,54 @@ bool CCoinsViewDB::BatchWrite(const CCoinsMap &mapCoins, const uint256 &hashBloc
    int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
    assert(!hashBlock.IsNull());

-    uint256 old_tip = GetBestBlock();
-    if (old_tip.IsNull()) {
-        // We may be in the middle of replaying.
-        std::vector<uint256> old_heads = GetHeadBlocks();
-        if (old_heads.size() == 2) {
-            assert(old_heads[0] == hashBlock);
-            old_tip = old_heads[1];
+    db << "BEGIN";
+    if (!mapCoins.empty()) {
+        db << "INSERT OR REPLACE INTO marker VALUES('head_block', ?)" << hashBlock;
+        auto dbd = db << "DELETE FROM unspent WHERE txID = ? AND txN = ?";
+        auto dbi = db << "INSERT OR REPLACE INTO unspent VALUES(?,?,?,?,?,?,?)";
+        for (auto it = mapCoins.begin(); it != mapCoins.end(); ++it) {
+            if (it->second.flags & CCoinsCacheEntry::DIRTY) {
+                if (it->second.coin.IsSpent()) {
+                    // at present the "IsSpent" flag is used for both "spent" and "block going backwards"
+                    dbd << it->first.hash << it->first.n;
+                    dbd++;
+                } else {
+                    CTxDestination address;
+                    std::string destination;
+                    if (ExtractDestination(it->second.coin.out.scriptPubKey, address))
+                        destination = EncodeDestination(address);
+                    uint32_t isCoinBase = it->second.coin.fCoinBase; // bit-field
+                    uint32_t coinHeight = it->second.coin.nHeight; // bit-field
+                    dbi << it->first.hash << it->first.n << isCoinBase << coinHeight
+                        << it->second.coin.out.nValue << it->second.coin.out.scriptPubKey << destination;
+                    dbi++;
+                }
+                changed++;
+            }
+            if (crash_simulate && ++count % 200000 == 0) {
+                static FastRandomContext rng;
+                if (rng.randrange(crash_simulate) == 0) {
+                    LogPrintf("Simulating a crash. Goodbye.\n");
+                    _Exit(0);
+                }
+            }
        }
+        dbd.used(true);
+        dbi.used(true);
+        db << "DELETE FROM marker WHERE name = 'head_block'";
    }
-
-    db << "begin";
-    db << "INSERT OR REPLACE INTO marker VALUES('head_block', ?)" << hashBlock;
-
-    auto dbd = db << "DELETE FROM unspent WHERE txID = ? AND txN = ?";
-    auto dbi = db << "INSERT OR REPLACE INTO unspent VALUES(?,?,?,?,?,?,?)";
-    for (auto it = mapCoins.begin(); it != mapCoins.end(); ++it) {
-        if (it->second.flags & CCoinsCacheEntry::DIRTY) {
-            if (it->second.coin.IsSpent()) {
-                // at present the "IsSpent" flag is used for both "spent" and "block going backwards"
-                dbd << it->first.hash << it->first.n;
-                dbd++;
-            }
-            else {
-                CTxDestination address;
-                std::string destination;
-                if (ExtractDestination(it->second.coin.out.scriptPubKey, address))
-                    destination = EncodeDestination(address);
-                uint32_t isCoinBase = it->second.coin.fCoinBase; // bit-field
-                uint32_t coinHeight = it->second.coin.nHeight; // bit-field
-                dbi << it->first.hash << it->first.n << isCoinBase << coinHeight
-                    << it->second.coin.out.nValue << it->second.coin.out.scriptPubKey << destination;
-                dbi++;
-            }
-            changed++;
-        }
-        if (crash_simulate && ++count % 200000 == 0) {
-            static FastRandomContext rng;
-            if (rng.randrange(crash_simulate) == 0) {
-                LogPrintf("Simulating a crash. Goodbye.\n");
-                _Exit(0);
-            }
-        }
-    }
-    dbd.used(true);
-    dbi.used(true);
    db << "INSERT OR REPLACE INTO marker VALUES('best_block', ?)" << hashBlock;
-    db << "DELETE FROM marker WHERE name = 'head_block'";

    auto code = sqlite::commit(db);
    if (code != SQLITE_OK) {
-        LogPrint(BCLog::COINDB, "Error committing transaction outputs changes to coin database. SQLite error: %d\n", code);
+        LogPrintf("%s: Error committing coins info to database. SQLite error: %d\n", __func__, code);
        return false;
    }
-    LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
+    LogPrint(BCLog::COINDB, "Committed %zu changed transaction outputs (out of %zu) to coin database...\n", changed, count);
    if (sync) {
-        auto code = sqlite::sync(db);
+        code = sqlite::sync(db);
        if (code != SQLITE_OK) {
-            LogPrint(BCLog::COINDB, "Error syncing coin database. SQLite error: %d\n", code);
+            LogPrintf("%s: Error syncing coin database. SQLite error: %d\n", __func__, code);
            return false;
        }
    }
@ -298,27 +288,33 @@ void CCoinsViewDBCursor::Next()
 bool CBlockTreeDB::BatchWrite(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo,
                              int nLastFile, const std::vector<const CBlockIndex*>& blockInfo, bool sync) {
    db << "begin";
-    auto ibf = db << "INSERT OR REPLACE INTO block_file(file, blocks, size, undoSize, heightFirst, heightLast, timeFirst, timeLast) "
-                     "VALUES(?,?,?,?,?,?,?,?)";
-    for (auto& kvp: fileInfo) {
-        ibf << kvp.first << kvp.second->nBlocks << kvp.second->nSize << kvp.second->nUndoSize
-            << kvp.second->nHeightFirst << kvp.second->nHeightLast << kvp.second->nTimeFirst << kvp.second->nTimeLast;
-        ibf++;
+    if (!fileInfo.empty()) {
+        auto ibf = db << "INSERT OR REPLACE INTO block_file(file, blocks, size, undoSize, heightFirst, "
+                         "heightLast, timeFirst, timeLast) VALUES(?,?,?,?,?,?,?,?)";
+        for (auto &kvp: fileInfo) {
+            ibf << kvp.first << kvp.second->nBlocks << kvp.second->nSize << kvp.second->nUndoSize
+                << kvp.second->nHeightFirst << kvp.second->nHeightLast << kvp.second->nTimeFirst
+                << kvp.second->nTimeLast;
+            ibf++;
+        }
+        ibf.used(true);
    }
-    ibf.used(true);
+
    db << "INSERT OR REPLACE INTO flag VALUES('last_block', ?)" << nLastFile; // TODO: is this always max(file column)?

-    auto ibi = db << "INSERT OR REPLACE INTO block_info(hash, prevHash, height, file, dataPos, undoPos, "
-                     "txCount, status, version, rootTxHash, rootTrieHash, time, bits, nonce) "
-                     "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
-    for (auto& bi: blockInfo) {
-        ibi << bi->GetBlockHash() << (bi->pprev ? bi->pprev->GetBlockHash() : uint256())
-            << bi->nHeight << bi->nFile << bi->nDataPos << bi->nUndoPos << bi->nTx
-            << bi->nStatus << bi->nVersion << bi->hashMerkleRoot << bi->hashClaimTrie
-            << bi->nTime << bi->nBits << bi->nNonce;
-        ibi++;
+    if(!blockInfo.empty()) {
+        auto ibi = db << "INSERT OR REPLACE INTO block_info(hash, prevHash, height, file, dataPos, undoPos, "
+                         "txCount, status, version, rootTxHash, rootTrieHash, time, bits, nonce) "
+                         "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+        for (auto &bi: blockInfo) {
+            ibi << bi->GetBlockHash() << (bi->pprev ? bi->pprev->GetBlockHash() : uint256())
+                << bi->nHeight << bi->nFile << bi->nDataPos << bi->nUndoPos << bi->nTx
+                << bi->nStatus << bi->nVersion << bi->hashMerkleRoot << bi->hashClaimTrie
+                << bi->nTime << bi->nBits << bi->nNonce;
+            ibi++;
+        }
+        ibi.used(true);
    }
-    ibi.used(true);
    auto code = sqlite::commit(db);
    if (code != SQLITE_OK) {
        LogPrintf("%s: Error committing block info to database. SQLite error: %d\n", __func__, code);
@ -326,7 +322,7 @@ bool CBlockTreeDB::BatchWrite(const std::vector<std::pair<int, const CBlockFileI
    }
    // by Sync they mean disk sync:
    if (sync) {
-        auto code = sqlite::sync(db);
+        code = sqlite::sync(db);
        if (code != SQLITE_OK) {
            LogPrintf("%s: Error syncing block database. SQLite error: %d\n", __func__, code);
            return false;
--- a/src/validation.cpp
+++ b/src/validation.cpp
@ -2412,7 +2412,8 @@ bool CChainState::FlushStateToDisk(
    const CChainParams& chainparams,
    CValidationState &state,
    FlushStateMode mode,
-    int nManualPruneHeight)
+    int nManualPruneHeight,
+    bool syncToDisk)
 {
    int64_t nMempoolUsage = mempool.DynamicMemoryUsage();
    LOCK(cs_main);
@ -2484,7 +2485,7 @@ bool CChainState::FlushStateToDisk(
                    vBlocks.push_back(*it);
                    setDirtyBlockIndex.erase(it++);
                }
-                if (!pblocktree->BatchWrite(vFiles, nLastBlockFile, vBlocks, mode == FlushStateMode::ALWAYS)) {
+                if (!pblocktree->BatchWrite(vFiles, nLastBlockFile, vBlocks, syncToDisk)) {
                    return AbortNode(state, "Failed to write to block index database");
                }
            }
@ -2503,10 +2504,10 @@ bool CChainState::FlushStateToDisk(
            if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * CoinsTip().GetCacheSize())) {
                return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX);
            }
-            if (mode == FlushStateMode::ALWAYS && !::Claimtrie().SyncToDisk())
+            if (syncToDisk && !::Claimtrie().SyncToDisk())
                return state.Error("Failed to write to claim trie database");
            // Flush the chainstate (which may refer to block index entries).
-            if (!CoinsTip().Flush(mode == FlushStateMode::ALWAYS))
+            if (!CoinsTip().Flush(syncToDisk))
                return AbortNode(state, "Failed to write to coin database");
            nLastFlush = nNow;
            full_flush_completed = true;
@ -2523,10 +2524,10 @@ bool CChainState::FlushStateToDisk(
    return true;
 }

-void CChainState::ForceFlushStateToDisk() {
+void CChainState::ForceFlushStateToDisk(bool syncToDisk) {
    CValidationState state;
    const CChainParams& chainparams = Params();
-    if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) {
+    if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS, 0, syncToDisk)) {
        LogPrintf("%s: failed to flush state (%s)\n", __func__, FormatStateMessage(state));
    }
 }
@ -2652,7 +2653,7 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha
    }
    LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI);
    // Write the chain state to disk, if necessary.
-    if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED))
+    if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
        return false;

    if (disconnectpool) {
@ -2796,7 +2797,7 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp
    int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3;
    LogPrint(BCLog::BENCH, "  - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal);
    // Write the chain state to disk, if necessary.
-    if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED))
+    if (!IsInitialBlockDownload() && !FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS))
        return false;
    int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4;
    LogPrint(BCLog::BENCH, "  - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal);
@ -3077,11 +3078,11 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
            if (!blocks_connected) return true;

            const CBlockIndex* pindexFork = m_chain.FindFork(starting_tip);
-            bool fInitialDownload = IsInitialBlockDownload();

            // Notify external listeners about the new tip.
            // Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected
            if (pindexFork != pindexNewTip) {
+                bool fInitialDownload = IsInitialBlockDownload();
                // Notify ValidationInterface subscribers
                GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload);

@ -3102,14 +3103,13 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams&
    } while (pindexNewTip != pindexMostWork);
    CheckBlockIndex(chainparams.GetConsensus());

-    auto flushMode = FlushStateMode::PERIODIC;
-    if (pindexNewTip && pindexNewTip->nTime + chainparams.GetConsensus().nPowTargetSpacing > GetAdjustedTime()) {
-        // trying to ensure that we flush to disk after new blocks when we're caught up to the chain
-        // they're technically allowed to be two hours late, but experience says one minute is more likely
-        // LogPrintf("Added tip with time %d but it is now %ll\n", pindexNewTip->nTime, GetAdjustedTime());
-        flushMode = FlushStateMode::ALWAYS;
-    }
-    return FlushStateToDisk(chainparams, state, flushMode);
+    auto& consensus = chainparams.GetConsensus();
+    CheckBlockIndex(consensus);
+
+    auto flushMode = IsInitialBlockDownload() ? FlushStateMode::IF_NEEDED : FlushStateMode::ALWAYS;
+    auto diskSync = chainparams.NetworkIDString() != CBaseChainParams::REGTEST
+                    && flushMode == FlushStateMode::ALWAYS;
+    return FlushStateToDisk(chainparams, state, flushMode, 0, diskSync);
 }

 bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) {
--- a/src/validation.h
+++ b/src/validation.h
@ -667,10 +667,11 @@ public:
        const CChainParams& chainparams,
        CValidationState &state,
        FlushStateMode mode,
-        int nManualPruneHeight = 0);
+        int nManualPruneHeight = 0,
+        bool syncToDisk=false);

    //! Unconditionally flush all changes to disk.
-    void ForceFlushStateToDisk();
+    void ForceFlushStateToDisk(bool syncToDisk=false);

    //! Prune blockfiles from the disk if necessary and then flush chainstate changes
    //! if we pruned.
--- a/src/wallet/rpcwallet.cpp
+++ b/src/wallet/rpcwallet.cpp
@ -3544,7 +3544,7 @@ static UniValue listunspent(const JSONRPCRequest& request)
        cctl.m_max_depth = nMaxDepth;
        auto locked_chain = pwallet->chain().lock();
        LOCK(pwallet->cs_wallet);
-        pwallet->AvailableCoins(*locked_chain, vecOutputs, !include_unsafe, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount);
+        pwallet->AvailableCoins(*locked_chain, vecOutputs, !include_unsafe, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, true);
    }

    LOCK(pwallet->cs_wallet);
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@ -1165,7 +1165,8 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose)
    }

    //// debug print
-    WalletLogPrintf("AddToWallet %s  %s%s\n", wtxIn.GetHash().ToString(), (fInsertedNew ? "new" : ""), (fUpdated ? "update" : ""));
+    if (LogAcceptCategory(BCLog::DB))
+        WalletLogPrintf("AddToWallet %s%s%s\n", wtxIn.GetHash().ToString(), (fInsertedNew ? "  new" : ""), (fUpdated ? "  update" : ""));

    // Write to disk
    if (fInsertedNew || fUpdated)
@ -2198,7 +2199,8 @@ bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, in
    if (GetDepthInMainChain(locked_chain) != 0) return false;

    // Submit transaction to mempool for relay
-    pwallet->WalletLogPrintf("Submitting wtx %s to mempool for relay\n", GetHash().ToString());
+    if (LogAcceptCategory(BCLog::DB))
+        pwallet->WalletLogPrintf("Submitting wtx %s to mempool for relay\n", GetHash().ToString());
    // We must set fInMempool here - while it will be re-set to true by the
    // entered-mempool callback, if we did not there would be a race where a
    // user could call sendmoney in a loop and hit spurious out of funds errors
@ -2487,7 +2489,7 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const
    return balance;
 }

-void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) const
+void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount, bool computeSolvable) const
 {
    AssertLockHeld(cs_wallet);

@ -2580,20 +2582,23 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<
                continue;
            }

-            if (!allow_used_addresses && IsUsedDestination(wtxid, i)) {
-                continue;
-            }
-            bool solvable = IsSolvable(*this,  wtx.tx->vout[i].scriptPubKey);
-            bool spendable = (mine & ISMINE_SPENDABLE) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable));
-
            // spending claims or supports requires specific selection:
-            auto isClaimCoin = (mine & ISMINE_CLAIM) || (mine & ISMINE_SUPPORT);
+            auto isClaimCoin = bool(mine & ISMINE_STAKE);
            auto claimSpendRequested = isClaimCoin && coinControl && coinControl->IsSelected(COutPoint(entry.first, i));
-
-            if (spendable && isClaimCoin && !claimSpendRequested)
+            if (isClaimCoin && !claimSpendRequested)
                continue;

-            vCoins.push_back(COutput(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly)));
+            bool solvable = false, computedSolvable = false;
+            bool spendable = bool(mine & ISMINE_SPENDABLE);
+            if (!spendable && bool(mine & ISMINE_WATCH_ONLY) && coinControl && coinControl->fAllowWatchOnly) {
+                solvable = IsSolvable(*this,  wtx.tx->vout[i].scriptPubKey); // this is a slow call
+                spendable = solvable;
+                computedSolvable = true;
+            };
+            if (computeSolvable && !computedSolvable)
+                solvable = IsSolvable(*this, wtx.tx->vout[i].scriptPubKey);
+
+            vCoins.emplace_back(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly));

            // Checks the sum amount of all UTXO's.
            if (nMinimumSumAmount != MAX_MONEY) {
@ -3700,7 +3705,8 @@ bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRe
        }

        m_pool_key_to_index.erase(keypool.vchPubKey.GetID());
-        WalletLogPrintf("keypool reserve %d\n", nIndex);
+        if (LogAcceptCategory(BCLog::DB))
+            WalletLogPrintf("keypool reserve %d\n", nIndex);
    }
    NotifyCanGetAddressesChanged();
    return true;
@ -3711,7 +3717,8 @@ void CWallet::KeepKey(int64_t nIndex)
    // Remove from key pool
    WalletBatch batch(*database);
    batch.ErasePool(nIndex);
-    WalletLogPrintf("keypool keep %d\n", nIndex);
+    if (LogAcceptCategory(BCLog::DB))
+        WalletLogPrintf("keypool keep %d\n", nIndex);
 }

 void CWallet::ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey)
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@ -675,8 +675,8 @@ public:
    int i;
    int nDepth;

-    /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */
-    int nInputBytes;
+    /** estimated size of this output as a fully-signed input in a transaction. Can be -1 */
+    mutable int nInputBytes;

    /** Whether we have the private keys to spend this output */
    bool fSpendable;
@ -694,20 +694,22 @@ public:
     */
    bool fSafe;

-    COutput(const CWalletTx *txIn, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn, bool fSafeIn, bool use_max_sig_in = false)
+    COutput(const CWalletTx *txIn, int iIn, int nDepthIn, bool fSpendableIn,
+            bool fSolvableIn, bool fSafeIn, bool use_max_sig_in = false)
+            : tx(txIn), i(iIn), nDepth(nDepthIn), fSpendable(fSpendableIn), fSolvable(fSolvableIn),
+            fSafe(fSafeIn), nInputBytes(-1), use_max_sig(use_max_sig_in)
    {
-        tx = txIn; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; fSafe = fSafeIn; nInputBytes = -1; use_max_sig = use_max_sig_in;
-        // If known and signable by the given wallet, compute nInputBytes
-        // Failure will keep this value -1
-        if (fSpendable && tx) {
-            nInputBytes = tx->GetSpendSize(i, use_max_sig);
-        }
    }

    std::string ToString() const;

    inline CInputCoin GetInputCoin() const
    {
+        // If known and signable by the given wallet, compute nInputBytes
+        // Failure will keep this value -1
+        if (nInputBytes < 0 && fSpendable && tx) {
+            nInputBytes = tx->GetSpendSize(i, use_max_sig);
+        }
        return CInputCoin(tx->tx, i, nInputBytes);
    }
 };
@ -981,7 +983,7 @@ public:
    /**
     * populate vCoins with vector of available COutputs.
     */
-    void AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe = true, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
+    void AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe = true, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0, bool computeSolvable=false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);

    /**
     * Return list of available coins and locked coins grouped by non-change output address.