Add hard fork to include more claim data

Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>

packaging/docker-for-binary/stuff/start.sh

@@ -35,6 +35,7 @@ function set_config() {
       override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
       override_config_option RPC_PORT rpcport $MERGED_CONFIG
       override_config_option RPC_BIND rpcbind $MERGED_CONFIG
+      override_config_option TX_INDEX txindex $MERGED_CONFIG
       override_config_option MAX_TX_FEE maxtxfee $MERGED_CONFIG
       override_config_option DUST_RELAY_FEE dustrelayfee $MERGED_CONFIG
       # Make the new merged config file the new CONFIG_PATH

src/Makefile.test.include

@@ -121,6 +121,7 @@ BITCOIN_TESTS =\
   test/claimtrieexpirationfork_tests.cpp \
   test/claimtriefixture.cpp \
   test/claimtriehashfork_tests.cpp \
+  test/claimtrieclaiminfo_tests.cpp \
   test/claimtrienormalization_tests.cpp \
   test/claimtrierpc_tests.cpp \
   test/nameclaim_tests.cpp \

src/chainparams.cpp

@@ -138,6 +138,7 @@ public:
         consensus.nMinRemovalWorkaroundHeight = 297706;
         consensus.nMaxRemovalWorkaroundHeight = 658300;
         consensus.nAllClaimsInMerkleForkHeight = 658310; // targeting 30 Oct 2019
+        consensus.nClaimInfoInMerkleForkHeight = 900000; // FIXME pick height
         consensus.fPowAllowMinDifficultyBlocks = false;
         consensus.fPowNoRetargeting = false;
         consensus.nRuleChangeActivationThreshold = 1916; // 95% of a  half week
@@ -247,6 +248,7 @@ public:
         consensus.nMinRemovalWorkaroundHeight = 99;
         consensus.nMaxRemovalWorkaroundHeight = 100;
         consensus.nAllClaimsInMerkleForkHeight = 110;
+        consensus.nClaimInfoInMerkleForkHeight = 1500000; // FIXME pick height
         consensus.fPowAllowMinDifficultyBlocks = true;
         consensus.fPowNoRetargeting = false;
         consensus.nRuleChangeActivationThreshold = 1512; // 75% for testchains
@@ -345,6 +347,7 @@ public:
         consensus.nMinRemovalWorkaroundHeight = -1;
         consensus.nMaxRemovalWorkaroundHeight = -1;
         consensus.nAllClaimsInMerkleForkHeight = 350;
+        consensus.nClaimInfoInMerkleForkHeight = 1350;
         consensus.fPowAllowMinDifficultyBlocks = false;
         consensus.fPowNoRetargeting = false;
         consensus.nRuleChangeActivationThreshold = 108; // 75% for testchains

src/claimscriptop.cpp

@@ -4,8 +4,13 @@
 
 #include <coins.h>
 #include <claimscriptop.h>
+#include <consensus/validation.h>
 #include <logging.h>
 #include <nameclaim.h>
+#include <validation.h>
+#include <util/translation.h>
+
+#include <boost/locale.hpp>
 
 CClaimScriptAddOp::CClaimScriptAddOp(const COutPoint& point, CAmount nValue, int nHeight)
     : point(point), nValue(nValue), nHeight(nHeight)
@@ -176,10 +181,49 @@ bool ProcessClaim(CClaimScriptOp& claimOp, CClaimTrieCache& trieCache, const CSc
         case OP_UPDATE_CLAIM:
             return claimOp.updateClaim(trieCache, vchToString(vvchParams[0]), uint160(vvchParams[1]));
         default:
-            throw std::runtime_error("Unimplemented OP handler.");
+            throw std::runtime_error(_("Unimplemented OP handler.").translated);
     }
 }
 
+static bool isUtf8(const std::string& name)
+{
+    using namespace boost::locale::conv;
+    try {
+        return to_utf<char>(name, "UTF-8", stop) == name;
+    } catch (const conversion_error&) {
+        return false;
+    }
+}
+
+bool ValidateClaimName(const CScript& scriptPubKey, std::string& reason)
+{
+    int op;
+    std::vector<std::vector<unsigned char> > vvchParams;
+    if (!DecodeClaimScript(scriptPubKey, op, vvchParams))
+        return true;
+
+    switch (op) {
+        case OP_CLAIM_NAME:
+        case OP_UPDATE_CLAIM:
+        case OP_SUPPORT_CLAIM:
+            break;
+        default:
+            reason = _("Unsupported operation").translated;
+            return false;
+    }
+    const auto name = vchToString(vvchParams[0]);
+    if (!isUtf8(name)) {
+        reason = _("Claim name is not valid UTF8 string").translated;
+        return false;
+    }
+    static const std::string disallowedSymbols("=&#:$@%?;/\\\"<>*\n\t\r\b\0", 20);
+    if (name.find_first_of(disallowedSymbols) != std::string::npos) {
+        reason = _("Claim name contains invalid symbol").translated;
+        return false;
+    }
+    return true;
+}
+
 void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoinsViewCache& view, int nHeight, const CUpdateCacheCallbacks& callbacks)
 {
     class CSpendClaimHistory : public CClaimScriptSpendOp

src/claimscriptop.h

@@ -242,4 +242,11 @@ struct CUpdateCacheCallbacks
  */
 void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoinsViewCache& view, int nHeight, const CUpdateCacheCallbacks& callbacks = {});
 
+/**
+ * Function to validate that we process only UTF8 claim names
+ * @param[in]  scriptPubKey     claim script to be decoded
+ * @param[in]  reason           rejection reason if any
+ * */
+bool ValidateClaimName(const CScript& scriptPubKey, std::string& reason);
+
 #endif // BITCOIN_CLAIMSCRIPTOP_H

src/claimtrie/data.cpp

@@ -2,6 +2,7 @@
 #include <data.h>
 
 #include <algorithm>
+#include <cassert>
 #include <sstream>
 
 CClaimValue::CClaimValue(COutPoint outPoint, uint160 claimId, int64_t nAmount, int nHeight, int nValidAtHeight)
@@ -121,3 +122,50 @@ CClaimTrieProofNode::CClaimTrieProofNode(std::vector<std::pair<unsigned char, ui
     : children(std::move(children)), hasValue(hasValue), valHash(std::move(valHash))
 {
 }
+
+// Sequence ordering is pretty similar to bid one (ASC)
+// what we have in CClaimValue::operator< just ignoring effective amount
+bool CClaimInfo::operator<(const CClaimInfo& other) const
+{
+    if (originalHeight < other.originalHeight)
+        return true;
+    if (originalHeight != other.originalHeight)
+        return false;
+    if (updateHeight > other.updateHeight)
+        return true;
+    if (updateHeight != other.updateHeight)
+        return false;
+    return outPoint != other.outPoint && !(outPoint < other.outPoint);
+}
+
+bool CClaimInfo::operator==(const CClaimInfo& other) const
+{
+    return claimId == other.claimId;
+}
+
+static CClaimInfo claimInfo(const CClaimNsupports& claimNsupports)
+{
+    CClaimInfo info;
+    info.originalHeight = claimNsupports.originalHeight;
+    info.updateHeight = claimNsupports.claim.nHeight;
+    info.outPoint = claimNsupports.claim.outPoint;
+    return info;
+}
+
+std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source)
+{
+    auto claimsNsupports = source;
+    std::sort(claimsNsupports.begin(), claimsNsupports.end(), [](const CClaimNsupports& lhs, const CClaimNsupports& rhs) {
+        return claimInfo(lhs) < claimInfo(rhs);
+    });
+    return claimsNsupports;
+}
+
+std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId)
+{
+    auto it = std::find_if(source.begin(), source.end(), [&claimId](const CClaimNsupports& claimNsupports) {
+        return claimNsupports.claim.claimId == claimId;
+    });
+    assert(it != source.end());
+    return std::distance(source.begin(), it);
+}

src/claimtrie/data.h

@@ -132,4 +132,19 @@ struct CClaimTrieProof
     COutPoint outPoint;
 };
 
+struct CClaimInfo
+{
+    CClaimInfo() = default;
+    bool operator<(const CClaimInfo& other) const;
+    bool operator==(const CClaimInfo& other) const;
+
+    uint160 claimId;
+    COutPoint outPoint;
+    int updateHeight = 0;
+    int originalHeight = 0;
+};
+
+std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source);
+std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId);
+
 #endif // CLAIMTRIE_DATA_H

src/claimtrie/forks.cpp

@@ -4,6 +4,10 @@
 #include <log.h>
 #include <trie.h>
 
+#include <iomanip>
+#include <sstream>
+#include <string>
+
 #include <boost/locale.hpp>
 #include <boost/locale/conversion.hpp>
 #include <boost/locale/localization_backend.hpp>
@@ -243,26 +247,67 @@ uint256 ComputeMerkleRoot(std::vector<uint256> hashes)
     return hashes.empty() ? uint256{} : hashes[0];
 }
 
+std::vector<uint256> CClaimTrieCacheHashFork::childrenHashes(const std::string& name, const std::function<void(const std::string&)>& callback)
+{
+    using row_type = sqlite::row_iterator::reference;
+    std::function<void(row_type)> visitor;
+    if (callback)
+        visitor = [&callback](row_type row) {
+            std::string childName;
+            row >> childName;
+            callback(childName);
+        };
+    else
+        visitor = [](row_type row) {
+            /* name */ row.index()++;
+        };
+    std::vector<uint256> childHashes;
+    for (auto&& row: childHashQuery << name) {
+        visitor(row);
+        row >> *childHashes.emplace(childHashes.end());
+    }
+    childHashQuery++;
+    return childHashes;
+}
+
+std::vector<uint256> CClaimTrieCacheHashFork::claimsHashes(const std::string& name, int takeoverHeight, const std::function<void(const CClaimInfo&)>& callback)
+{
+    using row_type = sqlite::row_iterator::reference;
+    std::function<COutPoint(row_type)> visitor;
+    if (callback)
+        visitor = [&callback](row_type row) -> COutPoint {
+            CClaimInfo info;
+            row >> info.outPoint.hash >> info.outPoint.n
+                >> info.claimId >> info.updateHeight;
+            // activationHeight, amount
+            row.index() += 2;
+            row >> info.originalHeight;
+            callback(info);
+            return info.outPoint;
+        };
+    else
+        visitor = [](row_type row) -> COutPoint {
+            COutPoint outPoint;
+            row >> outPoint.hash >> outPoint.n;
+            return outPoint;
+        };
+    std::vector<uint256> claimHashes;
+    for (auto&& row: claimHashQuery << nNextHeight << name)
+        claimHashes.push_back(getValueHash(visitor(row), takeoverHeight));
+    claimHashQuery++;
+    return claimHashes;
+}
+
 uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight)
 {
     if (nNextHeight < base->nAllClaimsInMerkleForkHeight)
         return CClaimTrieCacheNormalizationFork::computeNodeHash(name, claimsHash, takeoverHeight);
 
-    std::vector<uint256> childHashes;
+    auto childHashes = childrenHashes(name);
-    childHashQuery << name >> [&childHashes](std::string, uint256 hash) {
-        childHashes.push_back(std::move(hash));
-    };
-    childHashQuery++;
 
     if (takeoverHeight > 0) {
         if (claimsHash.IsNull()) {
-            COutPoint p;
+            auto hashes = claimsHashes(name, takeoverHeight);
-            std::vector<uint256> hashes;
-            for (auto&& row: claimHashQuery << nNextHeight << name) {
-                row >> p.hash >> p.n;
-                hashes.push_back(getValueHash(p, takeoverHeight));
-            }
-            claimHashQuery++;
             claimsHash = hashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(hashes));
         }
     } else {
@@ -275,7 +320,7 @@ uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, uint25
 
     const auto& childrenHash = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
 
-    return Hash(childrenHash.begin(), childrenHash.end(), claimsHash.begin(), claimsHash.end());
+    return Hash2(childrenHash, claimsHash);
 }
 
 std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
@@ -335,10 +380,10 @@ std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint3
 
 extern const std::string proofClaimQuery_s;
 
-bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uint160& claim, CClaimTrieProof& proof)
+bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uint160& claimId, CClaimTrieProof& proof)
 {
     if (nNextHeight < base->nAllClaimsInMerkleForkHeight)
-        return CClaimTrieCacheNormalizationFork::getProofForName(name, claim, proof);
+        return CClaimTrieCacheNormalizationFork::getProofForName(name, claimId, proof);
 
     auto fillPairs = [&proof](const std::vector<uint256>& hashes, uint32_t idx) {
         auto partials = ComputeMerklePath(hashes, idx);
@@ -353,32 +398,26 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uin
         std::string key;
         int takeoverHeight;
         row >> key >> takeoverHeight;
-        uint32_t nextCurrentIdx = 0;
-        std::vector<uint256> childHashes;
-        for (auto&& child : childHashQuery << key) {
-            std::string childKey;
-            uint256 childHash;
-            child >> childKey >> childHash;
-            if (name.find(childKey) == 0)
-                nextCurrentIdx = uint32_t(childHashes.size());
-            childHashes.push_back(childHash);
-        }
-        childHashQuery++;
 
-        std::vector<uint256> claimHashes;
+        uint32_t childIdx = 0, idx = 0;
-        uint32_t claimIdx = 0;
+        auto childHashes = childrenHashes(key, [&](const std::string& childKey) {
-        for (auto&& child: claimHashQuery << nNextHeight << key) {
+            if (name.find(childKey) == 0)
-            COutPoint childOutPoint;
+                childIdx = idx;
-            uint160 childClaimID;
+            ++idx;
-            child >> childOutPoint.hash >> childOutPoint.n >> childClaimID;
+        });
-            if (childClaimID == claim && key == name) {
+
-                claimIdx = uint32_t(claimHashes.size());
+        uint32_t claimIdx = 0; idx = 0;
-                proof.outPoint = childOutPoint;
+        std::function<void(const CClaimInfo&)> visitor;
-                proof.hasValue = true;
+        if (key == name)
-            }
+            visitor = [&](const CClaimInfo& info) {
-            claimHashes.push_back(getValueHash(childOutPoint, takeoverHeight));
+                if (info.claimId == claimId) {
-        }
+                    claimIdx = idx;
-        claimHashQuery++;
+                    proof.hasValue = true;
+                    proof.outPoint = info.outPoint;
+                }
+                ++idx;
+            };
+        auto claimHashes = claimsHashes(key, takeoverHeight, visitor);
 
         // I am on a node; I need a hash(children, claims)
         // if I am the last node on the list, it will be hash(children, x)
@@ -393,7 +432,7 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uin
             auto hash = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(claimHashes));
             proof.pairs.emplace_back(false, hash);
             if (!childHashes.empty())
-                fillPairs(childHashes, nextCurrentIdx);
+                fillPairs(childHashes, childIdx);
         }
     }
     std::reverse(proof.pairs.begin(), proof.pairs.end());
@@ -424,3 +463,72 @@ bool CClaimTrieCacheHashFork::allowSupportMetadata() const
 {
     return nNextHeight >= base->nAllClaimsInMerkleForkHeight;
 }
+
+CClaimTrieCacheClaimInfoHashFork::CClaimTrieCacheClaimInfoHashFork(CClaimTrie* base) : CClaimTrieCacheHashFork(base)
+{
+}
+
+extern std::vector<unsigned char> heightToVch(int n);
+
+// NOTE: the name is supposed to be the final one
+// normalized or not is the caller responsibility
+uint256 claimInfoHash(const std::string& name, const COutPoint& outPoint, int bid, int seq, int nHeightOfLastTakeover)
+{
+    auto hash = Hash2(Hash(name), Hash(outPoint.hash));
+    hash = Hash2(hash, std::to_string(outPoint.n));
+    hash = Hash2(hash, std::to_string(bid));
+    hash = Hash2(hash, std::to_string(seq));
+    return Hash2(hash, heightToVch(nHeightOfLastTakeover));
+}
+
+inline std::size_t indexOf(const std::vector<CClaimInfo>& infos, const CClaimInfo& info)
+{
+    return std::distance(infos.begin(), std::find(infos.begin(), infos.end(), info));
+}
+
+std::vector<uint256> CClaimTrieCacheClaimInfoHashFork::claimsHashes(const std::string& name, int takeoverHeight, const std::function<void(const CClaimInfo&)>& callback)
+{
+    if (nNextHeight < base->nClaimInfoInMerkleForkHeight)
+        return CClaimTrieCacheHashFork::claimsHashes(name, takeoverHeight, callback);
+
+    std::vector<CClaimInfo> claimsByBid;
+    for (auto&& row: claimHashQuery << nNextHeight << name) {
+        auto& cb = *claimsByBid.emplace(claimsByBid.end());
+        row >> cb.outPoint.hash >> cb.outPoint.n
+            >> cb.claimId >> cb.updateHeight;
+        // activationHeight, amount
+        row.index() += 2;
+        row >> cb.originalHeight;
+        if (callback) callback(cb);
+    }
+    claimHashQuery++;
+
+    auto claimsBySeq = claimsByBid;
+    std::sort(claimsBySeq.begin(), claimsBySeq.end());
+    std::vector<uint256> claimsHashes;
+    for (auto i = 0u; i < claimsByBid.size(); ++i) {
+        auto& cb = claimsByBid[i];
+        claimsHashes.push_back(claimInfoHash(name, cb.outPoint, i, indexOf(claimsBySeq, cb), takeoverHeight));
+    }
+    return claimsHashes;
+}
+
+void CClaimTrieCacheClaimInfoHashFork::initializeIncrement()
+{
+    CClaimTrieCacheHashFork::initializeIncrement();
+    // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
+    if (nNextHeight == base->nClaimInfoInMerkleForkHeight - 1) {
+        ensureTransacting();
+        db << "UPDATE node SET hash = NULL";
+    }
+}
+
+bool CClaimTrieCacheClaimInfoHashFork::finalizeDecrement()
+{
+    auto ret = CClaimTrieCacheHashFork::finalizeDecrement();
+    if (ret && nNextHeight == base->nClaimInfoInMerkleForkHeight - 1) {
+        ensureTransacting();
+        db << "UPDATE node SET hash = NULL";
+    }
+    return ret;
+}

src/claimtrie/forks.h

@@ -66,9 +66,27 @@ public:
     bool allowSupportMetadata() const;
 
 protected:
+    virtual std::vector<uint256> childrenHashes(const std::string& name,
+                                                const std::function<void(const std::string&)>& callback = {});
+    virtual std::vector<uint256> claimsHashes(const std::string& name, int takeoverHeight,
+                                              const std::function<void(const CClaimInfo&)>& callback = {});
     uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight) override;
 };
 
-typedef CClaimTrieCacheHashFork CClaimTrieCache;
+class CClaimTrieCacheClaimInfoHashFork : public CClaimTrieCacheHashFork
+{
+public:
+    explicit CClaimTrieCacheClaimInfoHashFork(CClaimTrie* base);
+    CClaimTrieCacheClaimInfoHashFork(CClaimTrieCacheClaimInfoHashFork&&) = default;
+
+    void initializeIncrement() override;
+    bool finalizeDecrement() override;
+
+protected:
+    std::vector<uint256> claimsHashes(const std::string& name, int takeoverHeight,
+                                      const std::function<void(const CClaimInfo&)>& callback = {}) override;
+};
+
+typedef CClaimTrieCacheClaimInfoHashFork CClaimTrieCache;
 
 #endif // CLAIMTRIE_FORKS_H

src/claimtrie/hashes.h

@@ -28,6 +28,41 @@ uint256 Hash(TIterator begin, TIterator end, Args... args)
     return CalcHash(&sha, begin, end, args...);
 }
 
+template <typename T>
+struct IsHashable {
+    static const bool value = false;
+};
+
+template <typename T>
+struct IsHashable<std::vector<T>> {
+    static const bool value = true;
+};
+
+template <>
+struct IsHashable<std::string> {
+    static const bool value = true;
+};
+
+template <>
+struct IsHashable<uint256> {
+    static const bool value = true;
+};
+
+template <typename T>
+uint256 Hash(const T& c)
+{
+    static_assert(IsHashable<T>::value, "T is not hashable");
+    return Hash(c.begin(), c.end());
+}
+
+template <typename T1, typename T2>
+uint256 Hash2(const T1& c1, const T2& c2)
+{
+    static_assert(IsHashable<T1>::value, "T1 is not hashable");
+    static_assert(IsHashable<T2>::value, "T2 is not hashable");
+    return Hash(c1.begin(), c1.end(), c2.begin(), c2.end());
+}
+
 extern std::function<void(std::vector<uint256>&)> sha256n_way;
 
 #endif // CLAIMTRIE_HASHES_H

src/claimtrie/libclaimtrie.i

@@ -20,6 +20,10 @@
 
 %ignore CBaseBlob(CBaseBlob &&);
 %ignore CClaimNsupports(CClaimNsupports &&);
+%ignore CClaimTrieCacheHashFork(CClaimTrieCacheHashFork &&);
+%ignore CClaimTrieCacheExpirationFork(CClaimTrieCacheExpirationFork &&);
+%ignore CClaimTrieCacheNormalizationFork(CClaimTrieCacheNormalizationFork &&);
+%ignore CClaimTrieCacheClaimInfoHashFork(CClaimTrieCacheClaimInfoHashFork &&);
 %ignore CClaimTrieProof(CClaimTrieProof &&);
 %ignore CClaimTrieProofNode(CClaimTrieProofNode &&);
 %ignore CClaimValue(CClaimValue &&);
@@ -39,7 +43,7 @@
 %include "txoutpoint.h"
 %include "data.h"
 
-%rename(CClaimTrieCache) CClaimTrieCacheHashFork;
+%rename(CClaimTrieCache) CClaimTrieCacheClaimInfoHashFork;
 
 %include "trie.h"
 %include "forks.h"

src/claimtrie/takeoverworkarounds.h

@@ -452,4 +452,16 @@ const boost::container::flat_map<std::pair<int, std::string>, int> takeoverWorka
         {{658098, "le-temps-nous-glisse-entre-les-doigts"}, 600099},
 };
 
+inline bool isTakeoverWorkaroundActive(int nHeight, const std::string& name)
+{
+    // This is a super ugly hack to work around bug in old code.
+    // The bug: un/support a name then update it. This will cause its takeover height to be reset to current.
+    // This is because the old code with add to the cache without setting block originals when dealing in supports.
+    if (nHeight < 658300) {
+        auto wit = takeoverWorkarounds.find(std::make_pair(nHeight, name));
+        return wit != takeoverWorkarounds.end();
+    }
+    return false;
+}
+
 #endif // TAKEOVERWORKAROUNDS_H

src/claimtrie/trie.cpp

@@ -24,11 +24,9 @@ std::vector<unsigned char> heightToVch(int n)
 
 uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover)
 {
-    auto hash1 = Hash(outPoint.hash.begin(), outPoint.hash.end());
+    auto hash1 = Hash(outPoint.hash);
-    auto snOut = std::to_string(outPoint.n);
+    auto hash2 = Hash(std::to_string(outPoint.n));
-    auto hash2 = Hash(snOut.begin(), snOut.end());
+    auto hash3 = Hash(heightToVch(nHeightOfLastTakeover));
-    auto vchHash = heightToVch(nHeightOfLastTakeover);
-    auto hash3 = Hash(vchHash.begin(), vchHash.end());
     return Hash(hash1.begin(), hash1.end(), hash2.begin(), hash2.end(), hash3.begin(), hash3.end());
 }
 
@@ -56,6 +54,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
                        int64_t nExtendedClaimExpirationTime,
                        int64_t nExtendedClaimExpirationForkHeight,
                        int64_t nAllClaimsInMerkleForkHeight,
+                       int64_t nClaimInfoInMerkleForkHeight,
                        int proportionalDelayFactor) :
                        nNextHeight(height),
                        dbCacheBytes(cacheBytes),
@@ -67,7 +66,8 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
                        nOriginalClaimExpirationTime(nOriginalClaimExpirationTime),
                        nExtendedClaimExpirationTime(nExtendedClaimExpirationTime),
                        nExtendedClaimExpirationForkHeight(nExtendedClaimExpirationForkHeight),
-                       nAllClaimsInMerkleForkHeight(nAllClaimsInMerkleForkHeight)
+                       nAllClaimsInMerkleForkHeight(nAllClaimsInMerkleForkHeight),
+                       nClaimInfoInMerkleForkHeight(nClaimInfoInMerkleForkHeight)
 {
     applyPragmas(db, cacheBytes >> 10U); // in KB
 
@@ -582,7 +582,7 @@ bool CClaimTrieCacheBase::flush()
 const std::string childHashQuery_s = "SELECT name, hash FROM node WHERE parent = ? ORDER BY name";
 
 const std::string claimHashQuery_s =
-    "SELECT c.txID, c.txN, c.claimID, c.updateHeight, c.activationHeight, c.amount, "
+    "SELECT c.txID, c.txN, c.claimID, c.updateHeight, c.activationHeight, c.amount, c.originalHeight, "
     "(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM support s INDEXED BY support_supportedClaimID "
     "WHERE s.supportedClaimID = c.claimID AND s.nodeName = c.nodeName "
     "AND s.activationHeight < ?1 AND s.expirationHeight >= ?1) as effectiveAmount "
@@ -798,9 +798,6 @@ bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string&
     return true;
 }
 
-// hardcoded claims that should trigger a takeover
-#include <takeoverworkarounds.h>
-
 bool CClaimTrieCacheBase::incrementBlock()
 {
     // the plan:
@@ -826,7 +823,11 @@ bool CClaimTrieCacheBase::incrementBlock()
     return true;
 }
 
-void CClaimTrieCacheBase::insertTakeovers(bool allowReplace) {
+// hardcoded claims that should trigger a takeover
+#include <takeoverworkarounds.h>
+
+void CClaimTrieCacheBase::insertTakeovers(bool allowReplace)
+{
     auto insertTakeoverQuery = allowReplace ?
             db << "INSERT OR REPLACE INTO takeover(name, height, claimID) VALUES(?, ?, ?)" :
             db << "INSERT INTO takeover(name, height, claimID) VALUES(?, ?, ?)";
@@ -847,13 +848,7 @@ void CClaimTrieCacheBase::insertTakeovers(bool allowReplace) {
         if (takeoverHappening && activateAllFor(nameWithTakeover))
             hasCandidate = getInfoForName(nameWithTakeover, candidateValue, 1);
 
-        // This is a super ugly hack to work around bug in old code.
+        takeoverHappening |= isTakeoverWorkaroundActive(nNextHeight, nameWithTakeover);
-        // The bug: un/support a name then update it. This will cause its takeover height to be reset to current.
-        // This is because the old code with add to the cache without setting block originals when dealing in supports.
-        if (nNextHeight < 658300) {
-            auto wit = takeoverWorkarounds.find(std::make_pair(nNextHeight, nameWithTakeover));
-            takeoverHappening |= wit != takeoverWorkarounds.end();
-        }
 
         logPrint << "Takeover on " << nameWithTakeover << " at " << nNextHeight << ", happening: " << takeoverHappening << ", set before: " << hasCurrentWinner << Clog::endl;
 

src/claimtrie/trie.h

@@ -18,18 +18,23 @@ uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover);
 
 class CClaimTrie
 {
+    // for unit tests
     friend class CClaimTrieCacheBase;
     friend class ClaimTrieChainFixture;
+    friend class ValidationBlockTests;
+
+    // forks have to be friends of trie
     friend class CClaimTrieCacheHashFork;
     friend class CClaimTrieCacheExpirationFork;
     friend class CClaimTrieCacheNormalizationFork;
-    friend class ValidationBlockTests;
+    friend class CClaimTrieCacheClaimInfoHashFork;
 
 public:
     CClaimTrie() = delete;
     CClaimTrie(CClaimTrie&&) = delete;
     CClaimTrie(const CClaimTrie&) = delete;
-    CClaimTrie(std::size_t cacheBytes, bool fWipe, int height = 0,
+    CClaimTrie(std::size_t cacheBytes,
+               bool fWipe, int height = 0,
                const std::string& dataDir = ".",
                int nNormalizedNameForkHeight = 1,
                int nMinRemovalWorkaroundHeight = 1,
@@ -38,6 +43,7 @@ public:
                int64_t nExtendedClaimExpirationTime = 1,
                int64_t nExtendedClaimExpirationForkHeight = 1,
                int64_t nAllClaimsInMerkleForkHeight = 1,
+               int64_t nClaimInfoInMerkleForkHeight = 1,
                int proportionalDelayFactor = 32);
 
     CClaimTrie& operator=(CClaimTrie&&) = delete;
@@ -62,6 +68,7 @@ protected:
     const int64_t nExtendedClaimExpirationTime;
     const int64_t nExtendedClaimExpirationForkHeight;
     const int64_t nAllClaimsInMerkleForkHeight;
+    const int64_t nClaimInfoInMerkleForkHeight;
 
 private:
     void doNodeTableMigration();
@@ -97,7 +104,7 @@ public:
                     int nHeight, int nValidHeight = -1);
 
     bool removeClaim(const uint160& claimId, const COutPoint& outPoint, std::string& nodeName,
-            int& validHeight, int& originalHeight);
+                     int& validHeight, int& originalHeight);
     bool removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight);
 
     virtual bool incrementBlock();
@@ -125,7 +132,6 @@ protected:
     int nNextHeight; // Height of the block that is being worked on, which is
     CClaimTrie* base;
     sqlite::database db;
-    mutable std::unordered_set<std::string> removalWorkaround;
     sqlite::database_binder childHashQuery, claimHashQuery, claimHashQueryLimit;
 
     virtual uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight);
@@ -136,10 +142,12 @@ protected:
     bool deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims);
     void ensureTreeStructureIsUpToDate();
     void ensureTransacting();
-    void insertTakeovers(bool allowReplace=false);
+    void insertTakeovers(bool allowReplace = false);
 
 private:
     bool transacting;
+    mutable std::unordered_set<std::string> removalWorkaround;
+
     // for unit test
     friend struct ClaimTrieChainFixture;
     friend class CClaimTrieCacheTest;

src/consensus/params.h

@@ -103,6 +103,7 @@ struct Params {
     }
     /** blocks before the hard fork that adds all claims into the merkle hash */
     int64_t nAllClaimsInMerkleForkHeight;
+    int64_t nClaimInfoInMerkleForkHeight;
     int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; }
     uint256 nMinimumChainWork;
     uint256 defaultAssumeValid;

src/policy/policy.cpp

@@ -9,7 +9,7 @@
 
 #include <consensus/validation.h>
 #include <coins.h>
-
+#include <claimscriptop.h>
 #include <nameclaim.h>
 
 
@@ -114,6 +114,9 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR
     unsigned int nDataOut = 0;
     txnouttype whichType;
     for (const CTxOut& txout : tx.vout) {
+        if (!ValidateClaimName(txout.scriptPubKey, reason))
+            return false;
+
         if (!::IsStandard(StripClaimScriptPrefix(txout.scriptPubKey), whichType)) {
             reason = "scriptpubkey";
             return false;

src/rpc/claimtrie.cpp

@@ -79,7 +79,8 @@ void RollBackTo(const CBlockIndex* targetIndex, CCoinsViewCache& coinsCache, CCl
     trieCache.getMerkleHash(); // update the hash tree
 }
 
-std::string escapeNonUtf8(const std::string& name)
+// it ensures name is utf8 like string
+std::string convertToUtf8(const std::string& name)
 {
     using namespace boost::locale::conv;
     try {
@@ -120,26 +121,6 @@ static bool extractValue(const CScript& scriptPubKey, std::string& name, std::st
     return true;
 }
 
-std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source)
-{
-    auto claimsNsupports = source;
-
-    std::sort(claimsNsupports.begin(), claimsNsupports.end(), [](const CClaimNsupports& lhs, const CClaimNsupports& rhs) {
-        return lhs.originalHeight < rhs.originalHeight || (lhs.originalHeight == rhs.originalHeight && lhs.claim < rhs.claim);
-    });
-
-    return claimsNsupports;
-}
-
-std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId)
-{
-    auto it = std::find_if(source.begin(), source.end(), [&claimId](const CClaimNsupports& claimNsupports) {
-        return claimNsupports.claim.claimId == claimId;
-    });
-    assert(it != source.end());
-    return std::distance(source.begin(), it);
-}
-
 UniValue claimToJSON(const CCoinsViewCache& coinsCache, const CClaimValue& claim)
 {
     UniValue result(UniValue::VOBJ);
@@ -148,7 +129,7 @@ UniValue claimToJSON(const CCoinsViewCache& coinsCache, const CClaimValue& claim
     if (!coin.IsSpent()) {
         std::string name, value;
         if (extractValue(coin.out.scriptPubKey, name, value)) {
-            result.pushKV(T_NAME, escapeNonUtf8(name));
+            result.pushKV(T_NAME, convertToUtf8(name));
             result.pushKV(T_VALUE, value);
         }
 
@@ -245,7 +226,7 @@ static UniValue getnamesintrie(const JSONRPCRequest& request)
 
     UniValue ret(UniValue::VARR);
     trieCache.getNamesInTrie([&ret](const std::string& name) {
-        ret.push_back(escapeNonUtf8(name));
+        ret.push_back(convertToUtf8(name));
 
         if (ShutdownRequested())
             throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested");
@@ -294,7 +275,7 @@ static UniValue getvalueforname(const JSONRPCRequest& request)
         bid = indexOf(csToName.claimsNsupports, claimIdIn);
     }
 
-    ret.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
+    ret.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
     ret.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
     ret.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
     ret.pushKV(T_BID, (int)bid);
@@ -320,7 +301,7 @@ UniValue getclaimsforname(const JSONRPCRequest& request)
     auto csToName = trieCache.getClaimsForName(name);
 
     UniValue result(UniValue::VOBJ);
-    result.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
+    result.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
 
     auto seqOrder = seqSort(csToName.claimsNsupports);
 
@@ -378,7 +359,7 @@ UniValue getclaimbybid(const JSONRPCRequest& request)
         seq = indexOf(seqOrder, claimNsupports.claim.claimId);
     }
 
-    result.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
+    result.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
     result.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
     result.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
     result.pushKV(T_BID, bid);
@@ -423,7 +404,7 @@ UniValue getclaimbyseq(const JSONRPCRequest& request)
         return claimNsupports;
     }();
 
-    result.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
+    result.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
     result.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
     result.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
     result.pushKV(T_BID, (int)bid);
@@ -458,7 +439,7 @@ UniValue getclaimbyid(const JSONRPCRequest& request)
                 seq = indexOf(seqOrder, foundClaim.claimId);
                 bid = indexOf(csToName.claimsNsupports, foundClaim.claimId);
             }
-            ret.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
+            ret.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
             ret.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
             ret.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
             ret.pushKV(T_BID, (int)bid);
@@ -529,7 +510,7 @@ UniValue getclaimsfortx(const JSONRPCRequest& request)
         UniValue o(UniValue::VOBJ);
         o.pushKV(T_N, static_cast<int64_t>(i));
         std::string sName(vvchParams[0].begin(), vvchParams[0].end());
-        o.pushKV(T_NAME, escapeNonUtf8(sName));
+        o.pushKV(T_NAME, convertToUtf8(sName));
         if (op == OP_CLAIM_NAME) {
             uint160 claimId = ClaimIdHash(hash, i);
             o.pushKV(T_CLAIMID, claimId.GetHex());

src/test/claimtrieclaiminfo_tests.cpp

@@ -0,0 +1,124 @@
+// Copyright (c) 2015-2019 The LBRY Foundation
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://opensource.org/licenses/mit-license.php
+
+#include <miner.h>
+#include <test/claimtriefixture.h>
+#include <validation.h>
+
+using namespace std;
+
+extern std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source);
+extern std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId);
+extern boost::test_tools::predicate_result ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash);
+extern uint256 claimInfoHash(const std::string& name, const COutPoint& outPoint, int bid, int seq, int nHeightOfLastTakeover);
+
+BOOST_FIXTURE_TEST_SUITE(claimtrieclaiminfo_tests, RegTestingSetup)
+
+BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_rollback_test)
+{
+    ClaimTrieChainFixture fixture;
+    fixture.setClaimInfoForkHeight(5);
+
+    CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 1);
+    fixture.IncrementBlocks(1);
+
+    auto currentRoot = fixture.getMerkleHash();
+    fixture.IncrementBlocks(1);
+    BOOST_CHECK_EQUAL(currentRoot, fixture.getMerkleHash());
+    fixture.IncrementBlocks(3);
+    BOOST_CHECK_NE(currentRoot, fixture.getMerkleHash());
+    fixture.DecrementBlocks(3);
+    BOOST_CHECK_EQUAL(currentRoot, fixture.getMerkleHash());
+}
+
+BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_single_test)
+{
+    ClaimTrieChainFixture fixture;
+    fixture.setClaimInfoForkHeight(2);
+    fixture.IncrementBlocks(4);
+
+    CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 1);
+    fixture.IncrementBlocks(1);
+
+    COutPoint outPoint(tx1.GetHash(), 0);
+    uint160 claimId = ClaimIdHash(tx1.GetHash(), 0);
+
+    CClaimTrieProof proof;
+    BOOST_CHECK(fixture.getProofForName("test", claimId, proof));
+    BOOST_CHECK(proof.hasValue);
+    BOOST_CHECK_EQUAL(proof.outPoint, outPoint);
+    auto claimHash = claimInfoHash("test", outPoint, 0, 0, proof.nHeightOfLastTakeover);
+    BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
+}
+
+BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_triple_test)
+{
+    ClaimTrieChainFixture fixture;
+    fixture.setClaimInfoForkHeight(2);
+    fixture.IncrementBlocks(4);
+
+    std::string names[] = {"test", "tester", "tester2"};
+    CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "one", 1);
+    CMutableTransaction tx2 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "two", 2);
+    CMutableTransaction tx3 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "thr", 3);
+    CMutableTransaction tx7 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "for", 4);
+    CMutableTransaction tx8 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "fiv", 5);
+    CMutableTransaction tx4 = fixture.MakeClaim(fixture.GetCoinbase(), names[1], "two", 2);
+    CMutableTransaction tx5 = fixture.MakeClaim(fixture.GetCoinbase(), names[1], "thr", 3);
+    CMutableTransaction tx6 = fixture.MakeClaim(fixture.GetCoinbase(), names[2], "one", 1);
+    fixture.IncrementBlocks(1);
+
+    for (const auto& name : names) {
+        int bid = 0;
+        auto cfn = fixture.getClaimsForName(name);
+        auto seqOrder = seqSort(cfn.claimsNsupports);
+        for (auto& claimSupports : cfn.claimsNsupports) {
+            CClaimTrieProof proof;
+            BOOST_CHECK_EQUAL(name, cfn.name); // normalization depends
+            auto& claim = claimSupports.claim;
+            BOOST_CHECK(fixture.getProofForName(name, claim.claimId, proof));
+            BOOST_CHECK(proof.hasValue);
+            BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
+            auto seq = indexOf(seqOrder, claim.claimId);
+            auto claimHash = claimInfoHash(name, claim.outPoint, bid++, seq, proof.nHeightOfLastTakeover);
+            BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
+        }
+    }
+}
+
+BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_branched_test)
+{
+    ClaimTrieChainFixture fixture;
+    fixture.setClaimInfoForkHeight(2);
+    fixture.IncrementBlocks(4);
+
+    std::string names[] = {"test", "toast", "tot", "top", "toa", "toad"};
+    for (const auto& name : names)
+        fixture.MakeClaim(fixture.GetCoinbase(), name, "one", 1);
+
+    fixture.MakeClaim(fixture.GetCoinbase(), "toa", "two", 2);
+    fixture.MakeClaim(fixture.GetCoinbase(), "toa", "tre", 3);
+    fixture.MakeClaim(fixture.GetCoinbase(), "toa", "qua", 4);
+    fixture.MakeClaim(fixture.GetCoinbase(), "toa", "cin", 5);
+    fixture.IncrementBlocks(1);
+
+    for (const auto& name : names) {
+        int bid = 0;
+        auto cfn = fixture.getClaimsForName(name);
+        auto seqOrder = seqSort(cfn.claimsNsupports);
+        for (auto& claimSupports : cfn.claimsNsupports) {
+            CClaimTrieProof proof;
+            BOOST_CHECK_EQUAL(name, cfn.name); // normalization depends
+            auto& claim = claimSupports.claim;
+            BOOST_CHECK(fixture.getProofForName(name, claim.claimId, proof));
+            BOOST_CHECK(proof.hasValue);
+            BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
+            auto seq = indexOf(seqOrder, claim.claimId);
+            auto claimHash = claimInfoHash(name, claim.outPoint, bid++, seq, proof.nHeightOfLastTakeover);
+            BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
+        }
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()

src/test/claimtriefixture.cpp

@@ -76,9 +76,10 @@ BlockAssembler AssemblerForTest()
 
 ClaimTrieChainFixture::ClaimTrieChainFixture() : CClaimTrieCache(&::Claimtrie()),
     unique_block_counter(0), normalization_original(-1), expirationForkHeight(-1), forkhash_original(-1),
-    minRemovalWorkaroundHeight(-1), maxRemovalWorkaroundHeight(-1)
+    claiminfo_original(-1), minRemovalWorkaroundHeight(-1), maxRemovalWorkaroundHeight(-1)
 {
-    fRequireStandard = false;
+    lRequireStandard = fRequireStandard;
+    fRequireStandard = false; // we don't require standard tx
     BOOST_CHECK_EQUAL(nNextHeight, ::ChainActive().Height() + 1);
     setNormalizationForkHeight(1000000);
 
@@ -98,6 +99,7 @@ ClaimTrieChainFixture::ClaimTrieChainFixture() : CClaimTrieCache(&::Claimtrie())
 ClaimTrieChainFixture::~ClaimTrieChainFixture()
 {
     added_unchecked = false;
+    fRequireStandard = lRequireStandard;
     DecrementBlocks(::ChainActive().Height());
     auto& consensus = const_cast<Consensus::Params&>(Params().GetConsensus());
     if (normalization_original >= 0) {
@@ -116,6 +118,10 @@ ClaimTrieChainFixture::~ClaimTrieChainFixture()
         consensus.nAllClaimsInMerkleForkHeight = forkhash_original;
         const_cast<int64_t&>(base->nAllClaimsInMerkleForkHeight) = forkhash_original;
     }
+    if (claiminfo_original >= 0) {
+        consensus.nClaimInfoInMerkleForkHeight = claiminfo_original;
+        const_cast<int64_t&>(base->nClaimInfoInMerkleForkHeight) = claiminfo_original;
+    }
     if (minRemovalWorkaroundHeight >= 0) {
         consensus.nMinRemovalWorkaroundHeight = minRemovalWorkaroundHeight;
         consensus.nMaxRemovalWorkaroundHeight = maxRemovalWorkaroundHeight;
@@ -124,7 +130,8 @@ ClaimTrieChainFixture::~ClaimTrieChainFixture()
     }
 }
 
-void ClaimTrieChainFixture::setRemovalWorkaroundHeight(int targetMinusCurrent, int blocks = 1000) {
+void ClaimTrieChainFixture::setRemovalWorkaroundHeight(int targetMinusCurrent, int blocks = 1000)
+{
     int target = ::ChainActive().Height() + targetMinusCurrent;
     auto& consensus = const_cast<Consensus::Params&>(Params().GetConsensus());
     if (minRemovalWorkaroundHeight < 0) {
@@ -174,6 +181,18 @@ void ClaimTrieChainFixture::setHashForkHeight(int targetMinusCurrent)
     const_cast<int64_t&>(base->nAllClaimsInMerkleForkHeight) = target;
 }
 
+void ClaimTrieChainFixture::setClaimInfoForkHeight(int targetMinusCurrent)
+{
+    if (targetMinusCurrent)
+        setHashForkHeight(targetMinusCurrent - 1);
+    int target = ::ChainActive().Height() + targetMinusCurrent;
+    auto& consensus = const_cast<Consensus::Params&>(Params().GetConsensus());
+    if (claiminfo_original < 0)
+        claiminfo_original = consensus.nClaimInfoInMerkleForkHeight;
+    consensus.nClaimInfoInMerkleForkHeight = target;
+    const_cast<int64_t&>(base->nClaimInfoInMerkleForkHeight) = target;
+}
+
 bool ClaimTrieChainFixture::CreateBlock(const std::unique_ptr<CBlockTemplate>& pblocktemplate)
 {
     CBlock* pblock = &pblocktemplate->block;

src/test/claimtriefixture.h

@@ -42,11 +42,13 @@ struct ClaimTrieChainFixture: public CClaimTrieCache
     int normalization_original;
     unsigned int num_txs_for_next_block;
     bool added_unchecked;
+    bool lRequireStandard;
 
     int64_t expirationForkHeight;
     int64_t originalExpiration;
     int64_t extendedExpiration;
     int64_t forkhash_original;
+    int64_t claiminfo_original;
     int minRemovalWorkaroundHeight, maxRemovalWorkaroundHeight;
 
     using CClaimTrieCache::getSupportsForName;
@@ -61,6 +63,8 @@ struct ClaimTrieChainFixture: public CClaimTrieCache
 
     void setHashForkHeight(int targetMinusCurrent);
 
+    void setClaimInfoForkHeight(int targetMinusCurrent);
+
     void setRemovalWorkaroundHeight(int targetMinusCurrent, int blocks);
 
     bool CreateBlock(const std::unique_ptr<CBlockTemplate>& pblocktemplate);

src/test/claimtriehashfork_tests.cpp

@@ -8,7 +8,7 @@
 
 using namespace std;
 
-void ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash)
+boost::test_tools::predicate_result ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash)
 {
     for (auto& pair : pairs)
         if (pair.first) // we're on the right because we were an odd index number
@@ -16,7 +16,12 @@ void ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uin
         else
             claimHash = Hash(claimHash.begin(), claimHash.end(), pair.second.begin(), pair.second.end());
 
-    BOOST_CHECK_EQUAL(cache.getMerkleHash(), claimHash);
+    if (cache.getMerkleHash() == claimHash)
+        return true;
+    boost::test_tools::predicate_result res(false);
+    res.message() << cache.getMerkleHash().ToString()
+                    << " != " << claimHash.ToString();
+    return res;
 }
 
 BOOST_FIXTURE_TEST_SUITE(claimtriehashfork_tests, RegTestingSetup)
@@ -55,7 +60,7 @@ BOOST_AUTO_TEST_CASE(hash_includes_all_claims_single_test)
     BOOST_CHECK(proof.hasValue);
     BOOST_CHECK_EQUAL(proof.outPoint, outPoint);
     auto claimHash = getValueHash(outPoint, proof.nHeightOfLastTakeover);
-    ValidatePairs(fixture, proof.pairs, claimHash);
+    BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
 }
 
 BOOST_AUTO_TEST_CASE(hash_includes_all_claims_triple_test)
@@ -84,7 +89,7 @@ BOOST_AUTO_TEST_CASE(hash_includes_all_claims_triple_test)
             BOOST_CHECK(proof.hasValue);
             BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
             auto claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover);
-            ValidatePairs(fixture, proof.pairs, claimHash);
+            BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
         }
     }
 }
@@ -113,7 +118,7 @@ BOOST_AUTO_TEST_CASE(hash_includes_all_claims_branched_test)
             BOOST_CHECK(proof.hasValue);
             BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
             auto claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover);
-            ValidatePairs(fixture, proof.pairs, claimHash);
+            BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
         }
     }
 }
@@ -174,7 +179,7 @@ BOOST_AUTO_TEST_CASE(hash_claims_children_fuzzer_test)
             BOOST_CHECK(proof.hasValue);
             BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
             auto claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover);
-            ValidatePairs(fixture, proof.pairs, claimHash);
+            BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
         }
     }
 }

src/test/claimtrierpc_tests.cpp

@@ -8,7 +8,7 @@
 
 using namespace std;
 
-extern void ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash);
+extern boost::test_tools::predicate_result ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash);
 
 BOOST_FIXTURE_TEST_SUITE(claimtrierpc_tests, RegTestingSetup)
 
@@ -325,7 +325,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
 
     result = getnameproof(req);
     auto claimHash = getValueHash(COutPoint(tx2.GetHash(), 1), result[T_LASTTAKEOVERHEIGHT].get_int());
-    ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
+    BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
 
     // check by partial id (can be even 2 chars)
     req.params = UniValue(UniValue::VARR);
@@ -335,7 +335,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
 
     result = getnameproof(req);
     claimHash = getValueHash(COutPoint(tx2.GetHash(), 0), result[T_LASTTAKEOVERHEIGHT].get_int());
-    ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
+    BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
 
     auto getclaimproofbybid = tableRPC["getclaimproofbybid"];
     req.params = UniValue(UniValue::VARR);
@@ -344,7 +344,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
 
     result = getclaimproofbybid(req);
     claimHash = getValueHash(COutPoint(tx1.GetHash(), 0), result[T_LASTTAKEOVERHEIGHT].get_int());
-    ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
+    BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
 
     auto getclaimproofbyseq = tableRPC["getclaimproofbyseq"];
     req.params = UniValue(UniValue::VARR);
@@ -353,7 +353,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
 
     result = getclaimproofbyseq(req);
     claimHash = getValueHash(COutPoint(tx3.GetHash(), 0), result[T_LASTTAKEOVERHEIGHT].get_int());
-    ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
+    BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
 
     auto getchangesinblock = tableRPC["getchangesinblock"];
     req.params = UniValue(UniValue::VARR);

src/validation.cpp

@@ -141,6 +141,7 @@ CClaimTrie& Claimtrie() {
                                 consensus.nExtendedClaimExpirationTime,
                                 consensus.nExtendedClaimExpirationForkHeight,
                                 consensus.nAllClaimsInMerkleForkHeight,
+                                consensus.nClaimInfoInMerkleForkHeight,
                                 Params().NetworkIDString() == CBaseChainParams::MAIN ? 32 : 1);
     };
     return *g_claimtrie;

src/wallet/rpcwallet.cpp

@@ -571,7 +571,7 @@ UniValue abandonclaim(const JSONRPCRequest& request)
 
 static void MaybePushAddress(UniValue& entry, const CTxDestination &dest);
 
-extern std::string escapeNonUtf8(const std::string&);
+extern std::string convertToUtf8(const std::string&);
 
 void ListNameClaims(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, CWallet* const pwallet, int nMinDepth,
                     UniValue& ret, const bool include_supports, bool list_spent)
@@ -599,7 +599,7 @@ void ListNameClaims(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx,
             continue;
 
         std::string sName (vvchParams[0].begin(), vvchParams[0].end());
-        entry.pushKV("name", escapeNonUtf8(sName));
+        entry.pushKV("name", convertToUtf8(sName));
         if (op == OP_CLAIM_NAME)
         {
             uint160 claimId = ClaimIdHash(wtx.GetHash(), s.vout);

src/wallet/test/claim_rpc_tests.cpp

@@ -526,5 +526,27 @@ BOOST_AUTO_TEST_CASE(remove_pruned_funds)
     PruneAbandonFunds(txid);
 }
 
+class HasMessage {
+public:
+    explicit HasMessage(const std::string& reason) : m_reason(reason) {}
+    bool operator() (const UniValue& e) const {
+        return e["message"].get_str().find(m_reason) != std::string::npos;
+    };
+private:
+    const std::string m_reason;
+};
+
+BOOST_AUTO_TEST_CASE(invalid_utf8_claim_name)
+{
+    // nClaimInfoInMerkleForkHeight = 1350
+    generateBlock(1350);
+    // disallow non UTF8 strings
+    BOOST_CHECK_EXCEPTION(ClaimAName("\xFF\xFF", "deadbeef", "1.0"), UniValue, HasMessage("Claim name is not valid UTF8 string"));
+    // disallow \0 in string
+    BOOST_CHECK_EXCEPTION(ClaimAName(std::string("test\0ab", 7), "deadbeef", "1.0"), UniValue, HasMessage("Claim name contains invalid symbol"));
+    // allow ^ in string
+    ClaimAName("test^ab", "deadbeef", "1.0");
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 

src/wallet/test/wallet_tests.cpp

@@ -340,8 +340,9 @@ BOOST_AUTO_TEST_CASE(LoadReceiveRequests)
 class ListCoinsTestingSetup : public TestChain100Setup
 {
 public:
-    ListCoinsTestingSetup()
+    ListCoinsTestingSetup() : lRequireStandard(fRequireStandard)
     {
+        fRequireStandard = false; // ensure all txs goes to wallet
         CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey()));
         wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock());
         bool firstRun;
@@ -359,6 +360,7 @@ public:
     ~ListCoinsTestingSetup()
     {
         wallet.reset();
+        fRequireStandard = lRequireStandard;
     }
 
     CWalletTx& AddTx(CRecipient recipient)
@@ -389,6 +391,7 @@ public:
         return it->second;
     }
 
+    bool lRequireStandard;
     std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain();
     std::unique_ptr<CWallet> wallet;
 };

src/wallet/wallet.cpp

@@ -15,7 +15,7 @@
 #include <key_io.h>
 #include <nameclaim.h>
 #include <policy/fees.h>
-#include <policy/policy.h>
+#include <policy/settings.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
 #include <script/descriptor.h>
@@ -3378,8 +3378,12 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve
 {
     {
         auto locked_chain = chain().lock();
-        LOCK(cs_wallet);
+        std::string reason;
+        // reject non standard tx before adding to mempool
+        if (fRequireStandard && !IsStandardTx(*tx, reason))
+            return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, reason);
 
+        LOCK(cs_wallet);
         CWalletTx wtxNew(this, std::move(tx));
         wtxNew.mapValue = std::move(mapValue);
         wtxNew.vOrderForm = std::move(orderForm);
@@ -3411,7 +3415,6 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve
             std::string err_string;
             if (!wtx.SubmitMemoryPoolAndRelay(err_string, true, *locked_chain)) {
                 WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", err_string);
-                // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure.
             }
         }
     }