Merge #11667: Add scripts to dumpwallet RPC

656fde5 Add script birthtime metadata to dump and import wallet (MeshCollider)
1bab9b2 Add script dump note to RPC help text and release notes (MeshCollider)
68c1e00 Add test for importwallet (MeshCollider)
9e1184d Add dumpwallet scripts test (MeshCollider)
ef0c730 Add scripts to importwallet RPC (MeshCollider)
b702ae8 Add CScripts to dumpwallet RPC (MeshCollider)
cdc260a Add GetCScripts to CBasicKeyStore (MeshCollider)

Pull request description:

  As discussed in https://github.com/bitcoin/bitcoin/pull/11289#issuecomment-334600457, adds the CScripts from the wallet to the `dumpwallet` RPC and then allows them to be imported with the `importwallet` RPC. Includes a basic test, and modifies the helptext of the dumpwallet RPC.

  Notes:
  - Reviewers: use `?w=1` to avoid the indentation-only change in commit `Add scripts to importwallet RPC `
  - currently the scripts are followed with `# addr=` comments just as the other keys are, unsure if this might confuse users into thinking all the scripts are for valid P2SH addresses though, but I don't think that should be an issue.
  - there are no birthtimes for scripts, so script imports don't affect rescans
  - `importwallet` imports the CScripts but I'm not sure how to approach specifying whether scripts are for P2SH addresses, BIP173 addresses, etc. whether that matters or not. Otherwise the RPC helptext might just need modification.

  Fixes #11715

Tree-SHA512: 36c55837b3a58b9d3499d4c0c2ae82153d62aa71919e751574651b63a1d2b8ecc83796db4553cc65dad9b5341c3a42ae2fcf4d62598c30af267f8e1461ba8272

doc/release-notes.md

@@ -100,6 +100,10 @@ The `share/rpcuser/rpcuser.py` script was renamed to `share/rpcauth/rpcauth.py`.
 used to create `rpcauth` credentials for a JSON-RPC user.
 
 
+- `dumpwallet` now includes hex-encoded scripts from the wallet in the dumpfile, and
+  `importwallet` now imports these scripts, but corresponding addresses may not be added
+  correctly or a manual rescan may be required to find relevant transactions.
+
 Credits
 =======
 

src/keystore.cpp

@@ -77,6 +77,16 @@ bool CBasicKeyStore::HaveCScript(const CScriptID& hash) const
     return mapScripts.count(hash) > 0;
 }
 
+std::set<CScriptID> CBasicKeyStore::GetCScripts() const
+{
+    LOCK(cs_KeyStore);
+    std::set<CScriptID> set_script;
+    for (const auto& mi : mapScripts) {
+        set_script.insert(mi.first);
+    }
+    return set_script;
+}
+
 bool CBasicKeyStore::GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const
 {
     LOCK(cs_KeyStore);

src/keystore.h

@@ -36,6 +36,7 @@ public:
     //! Support for BIP 0013 : see https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki
     virtual bool AddCScript(const CScript& redeemScript) =0;
     virtual bool HaveCScript(const CScriptID &hash) const =0;
+    virtual std::set<CScriptID> GetCScripts() const =0;
     virtual bool GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const =0;
 
     //! Support for Watch-only addresses
@@ -67,6 +68,7 @@ public:
     bool GetKey(const CKeyID &address, CKey &keyOut) const override;
     bool AddCScript(const CScript& redeemScript) override;
     bool HaveCScript(const CScriptID &hash) const override;
+    std::set<CScriptID> GetCScripts() const override;
     bool GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const override;
 
     bool AddWatchOnly(const CScript &dest) override;

src/wallet/rpcdump.cpp

@@ -500,40 +500,57 @@ UniValue importwallet(const JSONRPCRequest& request)
         if (vstr.size() < 2)
             continue;
         CBitcoinSecret vchSecret;
-        if (!vchSecret.SetString(vstr[0]))
+        if (vchSecret.SetString(vstr[0])) {
-            continue;
+            CKey key = vchSecret.GetKey();
-        CKey key = vchSecret.GetKey();
+            CPubKey pubkey = key.GetPubKey();
-        CPubKey pubkey = key.GetPubKey();
+            assert(key.VerifyPubKey(pubkey));
-        assert(key.VerifyPubKey(pubkey));
+            CKeyID keyid = pubkey.GetID();
-        CKeyID keyid = pubkey.GetID();
+            if (pwallet->HaveKey(keyid)) {
-        if (pwallet->HaveKey(keyid)) {
+                LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid));
-            LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid));
+                continue;
-            continue;
-        }
-        int64_t nTime = DecodeDumpTime(vstr[1]);
-        std::string strLabel;
-        bool fLabel = true;
-        for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) {
-            if (boost::algorithm::starts_with(vstr[nStr], "#"))
-                break;
-            if (vstr[nStr] == "change=1")
-                fLabel = false;
-            if (vstr[nStr] == "reserve=1")
-                fLabel = false;
-            if (boost::algorithm::starts_with(vstr[nStr], "label=")) {
-                strLabel = DecodeDumpString(vstr[nStr].substr(6));
-                fLabel = true;
             }
+            int64_t nTime = DecodeDumpTime(vstr[1]);
+            std::string strLabel;
+            bool fLabel = true;
+            for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) {
+                if (boost::algorithm::starts_with(vstr[nStr], "#"))
+                    break;
+                if (vstr[nStr] == "change=1")
+                    fLabel = false;
+                if (vstr[nStr] == "reserve=1")
+                    fLabel = false;
+                if (boost::algorithm::starts_with(vstr[nStr], "label=")) {
+                    strLabel = DecodeDumpString(vstr[nStr].substr(6));
+                    fLabel = true;
+                }
+            }
+            LogPrintf("Importing %s...\n", EncodeDestination(keyid));
+            if (!pwallet->AddKeyPubKey(key, pubkey)) {
+                fGood = false;
+                continue;
+            }
+            pwallet->mapKeyMetadata[keyid].nCreateTime = nTime;
+            if (fLabel)
+                pwallet->SetAddressBook(keyid, strLabel, "receive");
+            nTimeBegin = std::min(nTimeBegin, nTime);
+        } else if(IsHex(vstr[0])) {
+           std::vector<unsigned char> vData(ParseHex(vstr[0]));
+           CScript script = CScript(vData.begin(), vData.end());
+           if (pwallet->HaveCScript(script)) {
+               LogPrintf("Skipping import of %s (script already present)\n", vstr[0]);
+               continue;
+           }
+           if(!pwallet->AddCScript(script)) {
+               LogPrintf("Error importing script %s\n", vstr[0]);
+               fGood = false;
+               continue;
+           }
+           int64_t birth_time = DecodeDumpTime(vstr[1]);
+           if (birth_time > 0) {
+               pwallet->m_script_metadata[CScriptID(script)].nCreateTime = birth_time;
+               nTimeBegin = std::min(nTimeBegin, birth_time);
+           }
         }
-        LogPrintf("Importing %s...\n", EncodeDestination(keyid));
-        if (!pwallet->AddKeyPubKey(key, pubkey)) {
-            fGood = false;
-            continue;
-        }
-        pwallet->mapKeyMetadata[keyid].nCreateTime = nTime;
-        if (fLabel)
-            pwallet->SetAddressBook(keyid, strLabel, "receive");
-        nTimeBegin = std::min(nTimeBegin, nTime);
     }
     file.close();
     pwallet->ShowProgress("", 100); // hide progress dialog in GUI
@@ -542,7 +559,7 @@ UniValue importwallet(const JSONRPCRequest& request)
     pwallet->MarkDirty();
 
     if (!fGood)
-        throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys to wallet");
+        throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys/scripts to wallet");
 
     return NullUniValue;
 }
@@ -601,7 +618,7 @@ UniValue dumpwallet(const JSONRPCRequest& request)
         throw std::runtime_error(
             "dumpwallet \"filename\"\n"
             "\nDumps all wallet keys in a human-readable format to a server-side file. This does not allow overwriting existing files.\n"
-            "Imported scripts are not currently included in wallet dumps, these must be backed up separately.\n"
+            "Imported scripts are included in the dumpfile, but corresponding BIP173 addresses, etc. may not be added automatically by importwallet.\n"
             "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n"
             "only backing up the seed itself, and must be backed up too (e.g. ensure you back up the whole dumpfile).\n"
             "\nArguments:\n"
@@ -640,6 +657,9 @@ UniValue dumpwallet(const JSONRPCRequest& request)
     const std::map<CKeyID, int64_t>& mapKeyPool = pwallet->GetAllReserveKeys();
     pwallet->GetKeyBirthTimes(mapKeyBirth);
 
+    std::set<CScriptID> scripts = pwallet->GetCScripts();
+    // TODO: include scripts in GetKeyBirthTimes() output instead of separate
+
     // sort time/key pairs
     std::vector<std::pair<int64_t, CKeyID> > vKeyBirth;
     for (const auto& entry : mapKeyBirth) {
@@ -694,6 +714,21 @@ UniValue dumpwallet(const JSONRPCRequest& request)
         }
     }
     file << "\n";
+    for (const CScriptID &scriptid : scripts) {
+        CScript script;
+        std::string create_time = "0";
+        std::string address = EncodeDestination(scriptid);
+        // get birth times for scripts with metadata
+        auto it = pwallet->m_script_metadata.find(scriptid);
+        if (it != pwallet->m_script_metadata.end()) {
+            create_time = EncodeDumpTime(it->second.nCreateTime);
+        }
+        if(pwallet->GetCScript(scriptid, script)) {
+            file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time);
+            file << strprintf(" # addr=%s\n", address);
+        }
+    }
+    file << "\n";
     file << "# End of dump\n";
     file.close();
 

test/functional/wallet-dump.py

@@ -10,13 +10,14 @@ from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (assert_equal, assert_raises_rpc_error)
 
 
-def read_dump(file_name, addrs, hd_master_addr_old):
+def read_dump(file_name, addrs, script_addrs, hd_master_addr_old):
     """
     Read the given dump, count the addrs that match, count change and reserve.
     Also check that the old hd_master is inactive
     """
     with open(file_name, encoding='utf8') as inputfile:
         found_addr = 0
+        found_script_addr = 0
         found_addr_chg = 0
         found_addr_rsv = 0
         hd_master_addr_ret = None
@@ -38,6 +39,9 @@ def read_dump(file_name, addrs, hd_master_addr_old):
                         # ensure we have generated a new hd master key
                         assert(hd_master_addr_old != addr)
                         hd_master_addr_ret = addr
+                    elif keytype == "script=1":
+                        # scripts don't have keypaths
+                        keypath = None
                     else:
                         keypath = addr_keypath.rstrip().split("hdkeypath=")[1]
 
@@ -52,7 +56,14 @@ def read_dump(file_name, addrs, hd_master_addr_old):
                         elif keytype == "reserve=1":
                             found_addr_rsv += 1
                             break
-        return found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret
+
+                    # count scripts
+                    for script_addr in script_addrs:
+                        if script_addr == addr.rstrip() and keytype == "script=1":
+                            found_script_addr += 1
+                            break
+
+        return found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret
 
 
 class WalletDumpTest(BitcoinTestFramework):
@@ -81,13 +92,19 @@ class WalletDumpTest(BitcoinTestFramework):
         # Should be a no-op:
         self.nodes[0].keypoolrefill()
 
+        # Test scripts dump by adding a P2SH witness and a 1-of-1 multisig address
+        witness_addr = self.nodes[0].addwitnessaddress(addrs[0]["address"], True)
+        multisig_addr = self.nodes[0].addmultisigaddress(1, [addrs[1]["address"]])
+        script_addrs = [witness_addr, multisig_addr]
+
         # dump unencrypted wallet
         result = self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump")
         assert_equal(result['filename'], os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump"))
 
-        found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \
+        found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \
-            read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None)
+            read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, script_addrs, None)
         assert_equal(found_addr, test_addr_count)  # all keys must be in the dump
+        assert_equal(found_script_addr, 2)  # all scripts must be in the dump
         assert_equal(found_addr_chg, 50)  # 50 blocks where mined
         assert_equal(found_addr_rsv, 90*2) # 90 keys plus 100% internal keys
 
@@ -99,14 +116,29 @@ class WalletDumpTest(BitcoinTestFramework):
         self.nodes[0].keypoolrefill()
         self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.encrypted.dump")
 
-        found_addr, found_addr_chg, found_addr_rsv, _ = \
+        found_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \
-            read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, hd_master_addr_unenc)
+            read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, script_addrs, hd_master_addr_unenc)
         assert_equal(found_addr, test_addr_count)
+        assert_equal(found_script_addr, 2)
         assert_equal(found_addr_chg, 90*2 + 50)  # old reserve keys are marked as change now
         assert_equal(found_addr_rsv, 90*2) 
 
         # Overwriting should fail
         assert_raises_rpc_error(-8, "already exists", self.nodes[0].dumpwallet, tmpdir + "/node0/wallet.unencrypted.dump")
 
+        # Restart node with new wallet, and test importwallet
+        self.stop_node(0)
+        self.start_node(0, ['-wallet=w2'])
+
+        # Make sure the address is not IsMine before import
+        result = self.nodes[0].validateaddress(multisig_addr)
+        assert(result['ismine'] == False)
+
+        self.nodes[0].importwallet(os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump"))
+
+        # Now check IsMine is true
+        result = self.nodes[0].validateaddress(multisig_addr)
+        assert(result['ismine'] == True)
+
 if __name__ == '__main__':
     WalletDumpTest().main ()