LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Merge #15741: Batch write imported stuff in importmulti

Browse source 
0db94e55d wallet: Pass WalletBatch to CWallet::UnsetWalletFlag (João Barbosa)
6cb888b37 Apply the batch treatment to CWallet::SetAddressBook via ImportScriptPubKeys (Ben Woosley)
6154a09e0 Move some of ProcessImport into CWallet::Import* (Ben Woosley)
ccb26cf34 Batch writes for importmulti (Andrew Chow)
d6576e349 Have WalletBatch automatically flush every 1000 updates (Andrew Chow)
366fe0be0 Add AddWatchOnlyWithDB, AddKeyOriginWithDB, AddCScriptWithDB functions (Andrew Chow)

Pull request description:

  Instead of writing each item to the wallet database individually, do them in batches so that the import runs faster.

  This was tested by importing a ranged descriptor for 10,000 keys.

  Current master

  ```
  $ time src/bitcoin-cli -regtest -rpcwallet=importbig importmulti '[{"desc": "sh(wpkh([73111820/44h/1h/0h]tpubDDoT2SgEjaU5rerQpfcRDWPAcwyZ5g7xxHgVAfPwidgPDKVjm89d6jJ8AQotp35Np3m6VaysfUY1C2g68wFqUmraGbzhSsMF9YBuTGxpBaW/1/*))#3w7php47", "range": [0, 10000], "timestamp": "now", "internal": true, "keypool": false, "watchonly": true}]'
  ...

  real	7m45.29s
  ```

  This PR:

  ```
  $ time src/bitcoin-cli -regtest -rpcwallet=importbig4 importmulti '[{"desc": "pkh([73111820/44h/1h/0h]tpubDDoT2SgEjaU5rerQpfcRDWPAcwyZ5g7xxHgVAfPwidgPDKVjm89d6jJ8AQotp35Np3m6VaysfUY1C2g68wFqUmraGbzhSsMF9YBuTGxpBaW/1/*)#v65yjgmc", "range": [0, 10000], "timestamp": "now", "internal": true, "keypool": false, "watchonly": true}]'
  ...

  real	3.93s
  ```

  Fixes #15739

ACKs for commit 0db94e:
  jb55:
    utACK 0db94e5
  ariard:
    Tested ACK 0db94e5
  Empact:
    re-utACK 0db94e55dc only change is re the privacy of `UnsetWalletFlagWithDB` and `AddCScriptWithDB`.

Tree-SHA512: 3481308a64c99b6129f7bd328113dc291fe58743464628931feaebdef0e6ec770ddd5c19e4f9fbc1249a200acb04aaf62a8d914d53b0a29ac1e557576659c0cc
This commit is contained in:
MeshCollider 2019-05-29 18:54:07 +12:00
commit ed40fbb02a
No known key found for this signature in database
GPG key ID: D300116E1C875A3D
5 changed files with 163 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/wallet/db.cpp
View file

@ -607,7 +607,9 @@ void BerkeleyBatch::Flush()
if (fReadOnly)
nMinutes = 1;
env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault
env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
}
}
void BerkeleyDatabase::IncrementUpdateCounter()

54
src/wallet/rpcdump.cpp
View file

@ -1273,55 +1273,17 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con
// All good, time to import
pwallet->MarkDirty();
for (const auto& entry : import_data.import_scripts) {
if (!pwallet->HaveCScript(CScriptID(entry)) && !pwallet->AddCScript(entry)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding script to wallet");
}
}
for (const auto& entry : privkey_map) {
const CKey& key = entry.second;
CPubKey pubkey = key.GetPubKey();
const CKeyID& id = entry.first;
assert(key.VerifyPubKey(pubkey));
pwallet->mapKeyMetadata[id].nCreateTime = timestamp;
// If the private key is not present in the wallet, insert it.
if (!pwallet->HaveKey(id) && !pwallet->AddKeyPubKey(key, pubkey)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet");
}
pwallet->UpdateTimeFirstKey(timestamp);
if (!pwallet->ImportScripts(import_data.import_scripts)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding script to wallet");
}
for (const auto& entry : import_data.key_origins) {
pwallet->AddKeyOrigin(entry.second.first, entry.second.second);
if (!pwallet->ImportPrivKeys(privkey_map, timestamp)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet");
}
for (const CKeyID& id : ordered_pubkeys) {
auto entry = pubkey_map.find(id);
if (entry == pubkey_map.end()) {
continue;
}
const CPubKey& pubkey = entry->second;
CPubKey temp;
if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet");
}
pwallet->mapKeyMetadata[id].nCreateTime = timestamp;
// Add to keypool only works with pubkeys
if (add_keypool) {
pwallet->AddKeypoolPubkey(pubkey, internal);
}
if (!pwallet->ImportPubKeys(ordered_pubkeys, pubkey_map, import_data.key_origins, add_keypool, internal, timestamp)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet");
}
for (const CScript& script : script_pub_keys) {
if (!have_solving_data || !::IsMine(*pwallet, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated
if (!pwallet->AddWatchOnly(script, timestamp)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet");
}
}
CTxDestination dest;
ExtractDestination(script, dest);
if (!internal && IsValidDestination(dest)) {
pwallet->SetAddressBook(dest, label, "receive");
}
if (!pwallet->ImportScriptPubKeys(label, script_pub_keys, have_solving_data, internal, timestamp)) {
throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet");
}
result.pushKV("success", UniValue(true));

147
src/wallet/wallet.cpp
View file

@ -320,7 +320,7 @@ bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const C
secret.GetPrivKey(),
mapKeyMetadata[pubkey.GetID()]);
}
UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET);
UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET);
return true;
}
@ -362,12 +362,6 @@ void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata&
m_script_metadata[script_id] = meta;
}
// Writes a keymetadata for a public key. overwrite specifies whether to overwrite an existing metadata for that key if there exists one.
bool CWallet::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite)
{
return WalletBatch(*database).WriteKeyMetadata(meta, pubkey, overwrite);
}
void CWallet::UpgradeKeyMetadata()
{
AssertLockHeld(cs_wallet);
@ -376,7 +370,6 @@ void CWallet::UpgradeKeyMetadata()
}
std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(*database);
size_t cnt = 0;
for (auto& meta_pair : mapKeyMetadata) {
CKeyMetadata& meta = meta_pair.second;
if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin
@ -399,10 +392,6 @@ void CWallet::UpgradeKeyMetadata()
CPubKey pubkey;
if (GetPubKey(meta_pair.first, pubkey)) {
batch->WriteKeyMetadata(meta, pubkey, true);
if (++cnt % 1000 == 0) {
// avoid creating overlarge in-memory batches in case the wallet contains large amounts of keys
batch.reset(new WalletBatch(*database));
}
}
}
}
@ -432,11 +421,17 @@ void CWallet::UpdateTimeFirstKey(int64_t nCreateTime)
}
bool CWallet::AddCScript(const CScript& redeemScript)
{
WalletBatch batch(*database);
return AddCScriptWithDB(batch, redeemScript);
}
bool CWallet::AddCScriptWithDB(WalletBatch& batch, const CScript& redeemScript)
{
if (!CCryptoKeyStore::AddCScript(redeemScript))
return false;
if (WalletBatch(*database).WriteCScript(Hash160(redeemScript), redeemScript)) {
UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET);
if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) {
UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET);
return true;
}
return false;
@ -457,20 +452,32 @@ bool CWallet::LoadCScript(const CScript& redeemScript)
return CCryptoKeyStore::AddCScript(redeemScript);
}
bool CWallet::AddWatchOnly(const CScript& dest)
bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest)
{
if (!CCryptoKeyStore::AddWatchOnly(dest))
return false;
const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)];
UpdateTimeFirstKey(meta.nCreateTime);
NotifyWatchonlyChanged(true);
if (WalletBatch(*database).WriteWatchOnly(dest, meta)) {
UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET);
if (batch.WriteWatchOnly(dest, meta)) {
UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET);
return true;
}
return false;
}
bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time)
{
m_script_metadata[CScriptID(dest)].nCreateTime = create_time;
return AddWatchOnlyWithDB(batch, dest);
}
bool CWallet::AddWatchOnly(const CScript& dest)
{
WalletBatch batch(*database);
return AddWatchOnlyWithDB(batch, dest);
}
bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime)
{
m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime;
@ -1542,10 +1549,16 @@ void CWallet::SetWalletFlag(uint64_t flags)
}
void CWallet::UnsetWalletFlag(uint64_t flag)
{
WalletBatch batch(*database);
UnsetWalletFlagWithDB(batch, flag);
}
void CWallet::UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag)
{
LOCK(cs_wallet);
m_wallet_flags &= ~flag;
if (!WalletBatch(*database).WriteWalletFlags(m_wallet_flags))
if (!batch.WriteWalletFlags(m_wallet_flags))
throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed");
}
@ -1606,6 +1619,80 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut>
return true;
}
bool CWallet::ImportScripts(const std::set<CScript> scripts)
{
WalletBatch batch(*database);
for (const auto& entry : scripts) {
if (!HaveCScript(CScriptID(entry)) && !AddCScriptWithDB(batch, entry)) {
return false;
}
}
return true;
}
bool CWallet::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp)
{
WalletBatch batch(*database);
for (const auto& entry : privkey_map) {
const CKey& key = entry.second;
CPubKey pubkey = key.GetPubKey();
const CKeyID& id = entry.first;
assert(key.VerifyPubKey(pubkey));
mapKeyMetadata[id].nCreateTime = timestamp;
// If the private key is not present in the wallet, insert it.
if (!HaveKey(id) && !AddKeyPubKeyWithDB(batch, key, pubkey)) {
return false;
}
UpdateTimeFirstKey(timestamp);
}
return true;
}
bool CWallet::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp)
{
WalletBatch batch(*database);
for (const auto& entry : key_origins) {
AddKeyOriginWithDB(batch, entry.second.first, entry.second.second);
}
for (const CKeyID& id : ordered_pubkeys) {
auto entry = pubkey_map.find(id);
if (entry == pubkey_map.end()) {
continue;
}
const CPubKey& pubkey = entry->second;
CPubKey temp;
if (!GetPubKey(id, temp) && !AddWatchOnlyWithDB(batch, GetScriptForRawPubKey(pubkey), timestamp)) {
return false;
}
mapKeyMetadata[id].nCreateTime = timestamp;
// Add to keypool only works with pubkeys
if (add_keypool) {
AddKeypoolPubkeyWithDB(pubkey, internal, batch);
NotifyCanGetAddressesChanged();
}
}
return true;
}
bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool internal, const int64_t timestamp)
{
WalletBatch batch(*database);
for (const CScript& script : script_pub_keys) {
if (!have_solving_data || !::IsMine(*this, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated
if (!AddWatchOnlyWithDB(batch, script, timestamp)) {
return false;
}
}
CTxDestination dest;
ExtractDestination(script, dest);
if (!internal && IsValidDestination(dest)) {
SetAddressBookWithDB(batch, dest, label, "receive");
}
}
return true;
}
int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig)
{
std::vector<CTxOut> txouts;
@ -3149,8 +3236,7 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx)
return DBErrors::LOAD_OK;
}
bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose)
bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose)
{
bool fUpdated = false;
{
@ -3163,9 +3249,15 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s
}
NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO,
strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) );
if (!strPurpose.empty() && !WalletBatch(*database).WritePurpose(EncodeDestination(address), strPurpose))
if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose))
return false;
return WalletBatch(*database).WriteName(EncodeDestination(address), strName);
return batch.WriteName(EncodeDestination(address), strName);
}
bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose)
{
WalletBatch batch(*database);
return SetAddressBookWithDB(batch, address, strName, strPurpose);
}
bool CWallet::DelAddressBook(const CTxDestination& address)
@ -3315,13 +3407,6 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize)
return true;
}
void CWallet::AddKeypoolPubkey(const CPubKey& pubkey, const bool internal)
{
WalletBatch batch(*database);
AddKeypoolPubkeyWithDB(pubkey, internal, batch);
NotifyCanGetAddressesChanged();
}
void CWallet::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch)
{
LOCK(cs_wallet);
@ -4443,12 +4528,12 @@ bool CWallet::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const
return true;
}
bool CWallet::AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info)
bool CWallet::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info)
{
LOCK(cs_wallet);
std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint);
mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path;
mapKeyMetadata[pubkey.GetID()].has_key_origin = true;
mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path);
return WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true);
return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true);
}

33
src/wallet/wallet.h
View file

@ -775,6 +775,26 @@ private:
* nTimeFirstKey more intelligently for more efficient rescans.
*/
bool AddWatchOnly(const CScript& dest) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
/** Add a KeyOriginInfo to the wallet */
bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info);
//! Adds a key to the store, and saves it to disk.
bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
//! Adds a watch-only address to the store, and saves it to disk.
bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch);
bool SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose);
//! Adds a script to the store and saves it to disk
bool AddCScriptWithDB(WalletBatch& batch, const CScript& script);
//! Unsets a wallet flag and saves it to disk
void UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag);
/** Interface for accessing chain state. */
interfaces::Chain* m_chain;
@ -833,8 +853,6 @@ public:
// Map from Script ID to key metadata (for watch-only keys).
std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet);
bool WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, bool overwrite);
typedef std::map<unsigned int, CMasterKey> MasterKeyMap;
MasterKeyMap mapMasterKeys;
unsigned int nMasterKeyMaxID = 0;
@ -930,7 +948,6 @@ public:
CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
//! Adds a key to the store, and saves it to disk.
bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
//! Adds a key to the store, without saving it to disk (used by LoadWallet)
bool LoadKey(const CKey& key, const CPubKey &pubkey) { return CCryptoKeyStore::AddKeyPubKey(key, pubkey); }
//! Load metadata (used by LoadWallet)
@ -1049,6 +1066,11 @@ public:
bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig = false) const;
bool DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig = false) const;
bool ImportScripts(const std::set<CScript> scripts) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
bool ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE};
unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET};
bool m_spend_zero_conf_change{DEFAULT_SPEND_ZEROCONF_CHANGE};
@ -1070,8 +1092,6 @@ public:
bool NewKeyPool();
size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
bool TopUpKeyPool(unsigned int kpSize = 0);
void AddKeypoolPubkey(const CPubKey& pubkey, const bool internal);
void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch);
/**
* Reserves a key from the keypool and sets nIndex to its index
@ -1288,9 +1308,6 @@ public:
/** Implement lookup of key origin information through wallet key metadata. */
bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override;
/** Add a KeyOriginInfo to the wallet */
bool AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info);
};
/**

14
src/wallet/walletdb.h
View file

@ -143,9 +143,11 @@ public:
};
/** Access to the wallet database.
* This represents a single transaction at the
* database. It will be committed when the object goes out of scope.
* Optionally (on by default) it will flush to disk as well.
* Opens the database and provides read and write access to it. Each read and write is its own transaction.
* Multiple operation transactions can be started using TxnBegin() and committed using TxnCommit()
* Otherwise the transaction will be committed when the object goes out of scope.
* Optionally (on by default) it will flush to disk on close.
* Every 1000 writes will automatically trigger a flush to disk.
*/
class WalletBatch
{
@ -157,6 +159,9 @@ private:
return false;
}
m_database.IncrementUpdateCounter();
if (m_database.nUpdateCounter % 1000 == 0) {
m_batch.Flush();
}
return true;
}
@ -167,6 +172,9 @@ private:
return false;
}
m_database.IncrementUpdateCounter();
if (m_database.nUpdateCounter % 1000 == 0) {
m_batch.Flush();
}
return true;
}