waddrmgr: store watch-only accounts under new account type

Watch-only accounts are usually backed by an external signer as they do
not contain any private key information. Some external signers require a
root key fingerprint for identification and signing purposes. In order
to guarantee compatibility with external signers, we need to persist the
root key fingerprint within the database.

Before this change, watch-only accounts used the default account
database structure. In this commit, we introduce a new account type to
store different information for watch-only accounts only. This isn't a
breaking change as watch-only accounts have yet to be supported by the
primary user of the wallet (lnd). With this new account type, we can
avoid the empty private key fields, which are irrelevant to watch-only
accounts, and we can store the root key fingerprint.

waddrmgr/db.go

@@ -6,6 +6,7 @@
 package waddrmgr
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/binary"
 	"errors"
@@ -83,6 +84,12 @@ const (
 	// database. This is an account that re-uses the key derivation schema
 	// of BIP0044-like accounts.
 	accountDefault accountType = 0 // not iota as they need to be stable
+
+	// accountWatchOnly is the account type used for storing watch-only
+	// accounts within the database. This is an account that re-uses the key
+	// derivation schema of BIP0044-like accounts and does not store private
+	// keys.
+	accountWatchOnly accountType = 1
 )
 
 // dbAccountRow houses information stored about an account in the database.
@@ -102,6 +109,18 @@ type dbDefaultAccountRow struct {
 	name              string
 }
 
+// dbWatchOnlyAccountRow houses additional information stored about a watch-only
+// account in the databse.
+type dbWatchOnlyAccountRow struct {
+	dbAccountRow
+	pubKeyEncrypted      []byte
+	masterKeyFingerprint uint32
+	nextExternalIndex    uint32
+	nextInternalIndex    uint32
+	name                 string
+	addrSchema           *ScopeAddrSchema
+}
+
 // dbAddressRow houses common information stored about an address in the
 // database.
 type dbAddressRow struct {
@@ -809,6 +828,159 @@ func serializeDefaultAccountRow(encryptedPubKey, encryptedPrivKey []byte,
 	return rawData
 }
 
+// deserializeWatchOnlyAccountRow deserializes the raw data from the passed
+// account row as a watch-only account.
+func deserializeWatchOnlyAccountRow(accountID []byte,
+	row *dbAccountRow) (*dbWatchOnlyAccountRow, error) {
+
+	// The serialized BIP0044 watch-only account raw data format is:
+	//   <encpubkeylen><encpubkey><masterkeyfingerprint><nextextidx>
+	//   <nextintidx><namelen><name>
+	//
+	// 4 bytes encrypted pubkey len + encrypted pubkey + 4 bytes master key
+	// fingerprint + 4 bytes next external index + 4 bytes next internal
+	// index + 4 bytes name len + name + 1 byte addr schema exists + 2 bytes
+	// addr schema (if exists)
+
+	// Given the above, the length of the entry must be at a minimum
+	// the constant value sizes.
+	if len(row.rawData) < 21 {
+		str := fmt.Sprintf("malformed serialized watch-only account "+
+			"for key %x", accountID)
+		return nil, managerError(ErrDatabase, str, nil)
+	}
+
+	retRow := dbWatchOnlyAccountRow{
+		dbAccountRow: *row,
+	}
+	r := bytes.NewReader(row.rawData)
+
+	var pubLen uint32
+	err := binary.Read(r, binary.LittleEndian, &pubLen)
+	if err != nil {
+		return nil, err
+	}
+	retRow.pubKeyEncrypted = make([]byte, pubLen)
+	err = binary.Read(r, binary.LittleEndian, &retRow.pubKeyEncrypted)
+	if err != nil {
+		return nil, err
+	}
+
+	err = binary.Read(r, binary.LittleEndian, &retRow.masterKeyFingerprint)
+	if err != nil {
+		return nil, err
+	}
+
+	err = binary.Read(r, binary.LittleEndian, &retRow.nextExternalIndex)
+	if err != nil {
+		return nil, err
+	}
+	err = binary.Read(r, binary.LittleEndian, &retRow.nextInternalIndex)
+	if err != nil {
+		return nil, err
+	}
+
+	var nameLen uint32
+	err = binary.Read(r, binary.LittleEndian, &nameLen)
+	if err != nil {
+		return nil, err
+	}
+	name := make([]byte, nameLen)
+	err = binary.Read(r, binary.LittleEndian, &name)
+	if err != nil {
+		return nil, err
+	}
+	retRow.name = string(name)
+
+	var addrSchemaExists bool
+	err = binary.Read(r, binary.LittleEndian, &addrSchemaExists)
+	if err != nil {
+		return nil, err
+	}
+	if addrSchemaExists {
+		var addrSchemaBytes [2]byte
+		err = binary.Read(r, binary.LittleEndian, &addrSchemaBytes)
+		if err != nil {
+			return nil, err
+		}
+		retRow.addrSchema = scopeSchemaFromBytes(addrSchemaBytes[:])
+	}
+
+	return &retRow, nil
+}
+
+// serializeWatchOnlyAccountRow returns the serialization of the raw data field
+// for a watch-only account.
+func serializeWatchOnlyAccountRow(encryptedPubKey []byte, masterKeyFingerprint,
+	nextExternalIndex, nextInternalIndex uint32, name string,
+	addrSchema *ScopeAddrSchema) ([]byte, error) {
+
+	// The serialized BIP0044 account raw data format is:
+	//   <encpubkeylen><encpubkey><masterkeyfingerprint><nextextidx>
+	//   <nextintidx><namelen><name>
+	//
+	// 4 bytes encrypted pubkey len + encrypted pubkey + 4 bytes master key
+	// fingerprint + 4 bytes next external index + 4 bytes next internal
+	// index + 4 bytes name len + name + 1 byte addr schema exists + 2 bytes
+	// addr schema (if exists)
+	pubLen := uint32(len(encryptedPubKey))
+	nameLen := uint32(len(name))
+
+	addrSchemaExists := addrSchema != nil
+	var addrSchemaBytes []byte
+	if addrSchemaExists {
+		addrSchemaBytes = scopeSchemaToBytes(addrSchema)
+	}
+
+	bufLen := 21 + pubLen + nameLen + uint32(len(addrSchemaBytes))
+	buf := bytes.NewBuffer(make([]byte, 0, bufLen))
+
+	err := binary.Write(buf, binary.LittleEndian, pubLen)
+	if err != nil {
+		return nil, err
+	}
+	err = binary.Write(buf, binary.LittleEndian, encryptedPubKey)
+	if err != nil {
+		return nil, err
+	}
+
+	err = binary.Write(buf, binary.LittleEndian, masterKeyFingerprint)
+	if err != nil {
+		return nil, err
+	}
+
+	err = binary.Write(buf, binary.LittleEndian, nextExternalIndex)
+	if err != nil {
+		return nil, err
+	}
+	err = binary.Write(buf, binary.LittleEndian, nextInternalIndex)
+	if err != nil {
+		return nil, err
+	}
+
+	err = binary.Write(buf, binary.LittleEndian, nameLen)
+	if err != nil {
+		return nil, err
+	}
+	err = binary.Write(buf, binary.LittleEndian, []byte(name))
+	if err != nil {
+		return nil, err
+	}
+
+	err = binary.Write(buf, binary.LittleEndian, addrSchemaExists)
+	if err != nil {
+		return nil, err
+	}
+	if addrSchemaExists {
+		err = binary.Write(buf, binary.LittleEndian, addrSchemaBytes)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
 // forEachKeyScope calls the given function for each known manager scope
 // within the set of scopes known by the root manager.
 func forEachKeyScope(ns walletdb.ReadBucket, fn func(KeyScope) error) error {
@@ -947,6 +1119,8 @@ func fetchAccountInfo(ns walletdb.ReadBucket, scope *KeyScope,
 	switch row.acctType {
 	case accountDefault:
 		return deserializeDefaultAccountRow(accountID, row)
+	case accountWatchOnly:
+		return deserializeWatchOnlyAccountRow(accountID, row)
 	}
 
 	str := fmt.Sprintf("unsupported account type '%d'", row.acctType)
@@ -1087,8 +1261,9 @@ func putAccountRow(ns walletdb.ReadWriteBucket, scope *KeyScope,
 	return nil
 }
 
-// putAccountInfo stores the provided account information to the database.
+// putDefaultAccountInfo stores the provided default account information to the
-func putAccountInfo(ns walletdb.ReadWriteBucket, scope *KeyScope,
+// database.
+func putDefaultAccountInfo(ns walletdb.ReadWriteBucket, scope *KeyScope,
 	account uint32, encryptedPubKey, encryptedPrivKey []byte,
 	nextExternalIndex, nextInternalIndex uint32, name string) error {
 
@@ -1103,7 +1278,38 @@ func putAccountInfo(ns walletdb.ReadWriteBucket, scope *KeyScope,
 		acctType: accountDefault,
 		rawData:  rawData,
 	}
-	if err := putAccountRow(ns, scope, account, &acctRow); err != nil {
+	return putAccountInfo(ns, scope, account, &acctRow, name)
+}
+
+// putWatchOnlyAccountInfo stores the provided watch-only account information to
+// the database.
+func putWatchOnlyAccountInfo(ns walletdb.ReadWriteBucket, scope *KeyScope,
+	account uint32, encryptedPubKey []byte, masterKeyFingerprint,
+	nextExternalIndex, nextInternalIndex uint32, name string,
+	addrSchema *ScopeAddrSchema) error {
+
+	rawData, err := serializeWatchOnlyAccountRow(
+		encryptedPubKey, masterKeyFingerprint, nextExternalIndex,
+		nextInternalIndex, name, addrSchema,
+	)
+	if err != nil {
+		return err
+	}
+
+	// TODO(roasbeef): pass scope bucket directly??
+
+	acctRow := dbAccountRow{
+		acctType: accountWatchOnly,
+		rawData:  rawData,
+	}
+	return putAccountInfo(ns, scope, account, &acctRow, name)
+}
+
+// putAccountInfo stores the provided account information to the database.
+func putAccountInfo(ns walletdb.ReadWriteBucket, scope *KeyScope,
+	account uint32, acctRow *dbAccountRow, name string) error {
+
+	if err := putAccountRow(ns, scope, account, acctRow); err != nil {
 		return err
 	}
 
@@ -1113,11 +1319,7 @@ func putAccountInfo(ns walletdb.ReadWriteBucket, scope *KeyScope,
 	}
 
 	// Update account name index.
-	if err := putAccountNameIndex(ns, scope, account, name); err != nil {
+	return putAccountNameIndex(ns, scope, account, name)
-		return err
-	}
-
-	return nil
 }
 
 // putLastAccount stores the provided metadata - last account - to the
@@ -1479,32 +1681,64 @@ func putChainedAddress(ns walletdb.ReadWriteBucket, scope *KeyScope,
 	if err != nil {
 		return err
 	}
-	arow, err := deserializeDefaultAccountRow(accountID, row)
+
-	if err != nil {
+	switch row.acctType {
-		return err
+	case accountDefault:
+		arow, err := deserializeDefaultAccountRow(accountID, row)
+		if err != nil {
+			return err
+		}
+
+		// Increment the appropriate next index depending on whether the
+		// branch is internal or external.
+		nextExternalIndex := arow.nextExternalIndex
+		nextInternalIndex := arow.nextInternalIndex
+		if branch == InternalBranch {
+			nextInternalIndex = index + 1
+		} else {
+			nextExternalIndex = index + 1
+		}
+
+		// Reserialize the account with the updated index and store it.
+		row.rawData = serializeDefaultAccountRow(
+			arow.pubKeyEncrypted, arow.privKeyEncrypted,
+			nextExternalIndex, nextInternalIndex, arow.name,
+		)
+
+	case accountWatchOnly:
+		arow, err := deserializeWatchOnlyAccountRow(accountID, row)
+		if err != nil {
+			return err
+		}
+
+		// Increment the appropriate next index depending on whether the
+		// branch is internal or external.
+		nextExternalIndex := arow.nextExternalIndex
+		nextInternalIndex := arow.nextInternalIndex
+		if branch == InternalBranch {
+			nextInternalIndex = index + 1
+		} else {
+			nextExternalIndex = index + 1
+		}
+
+		// Reserialize the account with the updated index and store it.
+		row.rawData, err = serializeWatchOnlyAccountRow(
+			arow.pubKeyEncrypted, arow.masterKeyFingerprint,
+			nextExternalIndex, nextInternalIndex, arow.name,
+			arow.addrSchema,
+		)
+		if err != nil {
+			return err
+		}
 	}
 
-	// Increment the appropriate next index depending on whether the branch
-	// is internal or external.
-	nextExternalIndex := arow.nextExternalIndex
-	nextInternalIndex := arow.nextInternalIndex
-	if branch == InternalBranch {
-		nextInternalIndex = index + 1
-	} else {
-		nextExternalIndex = index + 1
-	}
-
-	// Reserialize the account with the updated index and store it.
-	row.rawData = serializeDefaultAccountRow(
-		arow.pubKeyEncrypted, arow.privKeyEncrypted, nextExternalIndex,
-		nextInternalIndex, arow.name,
-	)
 	err = bucket.Put(accountID, serializeAccountRow(row))
 	if err != nil {
 		str := fmt.Sprintf("failed to update next index for "+
 			"address %x, account %d", addressID, account)
 		return managerError(ErrDatabase, str, err)
 	}
+
 	return nil
 }
 
@@ -1741,6 +1975,9 @@ func deletePrivateKeys(ns walletdb.ReadWriteBucket) error {
 					str := "failed to delete account private key"
 					return managerError(ErrDatabase, str, err)
 				}
+
+			// Watch-only accounts don't contain any private keys.
+			case accountWatchOnly:
 			}
 
 			return nil

waddrmgr/manager.go

@@ -1678,7 +1678,7 @@ func createManagerKeyScope(ns walletdb.ReadWriteBucket,
 	}
 
 	// Save the information for the default account to the database.
-	err = putAccountInfo(
+	err = putDefaultAccountInfo(
 		ns, &scope, DefaultAccountNum, acctPubEnc, acctPrivEnc, 0, 0,
 		defaultAccountName,
 	)
@@ -1686,7 +1686,7 @@ func createManagerKeyScope(ns walletdb.ReadWriteBucket,
 		return err
 	}
 
-	return putAccountInfo(
+	return putDefaultAccountInfo(
 		ns, &scope, ImportedAddrAccount, nil, nil, 0, 0,
 		ImportedAddrAccountName,
 	)

waddrmgr/scoped_manager.go

@@ -309,71 +309,95 @@ func (s *ScopedKeyManager) loadAccountInfo(ns walletdb.ReadBucket,
 		return nil, maybeConvertDbError(err)
 	}
 
-	// Ensure the account type is a default account.
+	decryptKey := func(cryptoKey EncryptorDecryptor,
-	row, ok := rowInterface.(*dbDefaultAccountRow)
+		encryptedKey []byte) (*hdkeychain.ExtendedKey, error) {
-	if !ok {
-		str := fmt.Sprintf("unsupported account type %T", row)
-		return nil, managerError(ErrDatabase, str, nil)
-	}
 
-	// Use the crypto public key to decrypt the account public extended
+		serializedKey, err := cryptoKey.Decrypt(encryptedKey)
-	// key.
-	serializedKeyPub, err := s.rootManager.cryptoKeyPub.Decrypt(row.pubKeyEncrypted)
-	if err != nil {
-		str := fmt.Sprintf("failed to decrypt public key for account %d",
-			account)
-		return nil, managerError(ErrCrypto, str, err)
-	}
-	acctKeyPub, err := hdkeychain.NewKeyFromString(string(serializedKeyPub))
-	if err != nil {
-		str := fmt.Sprintf("failed to create extended public key for "+
-			"account %d", account)
-		return nil, managerError(ErrKeyChain, str, err)
-	}
-
-	// Create the new account info with the known information.  The rest of
-	// the fields are filled out below.
-	acctInfo := &accountInfo{
-		acctName:          row.name,
-		acctKeyEncrypted:  row.privKeyEncrypted,
-		acctKeyPub:        acctKeyPub,
-		nextExternalIndex: row.nextExternalIndex,
-		nextInternalIndex: row.nextInternalIndex,
-	}
-
-	watchOnly := s.rootManager.watchOnly() || len(acctInfo.acctKeyEncrypted) == 0
-	private := !s.rootManager.isLocked() && !watchOnly
-	if private {
-		// Use the crypto private key to decrypt the account private
-		// extended keys.
-		decrypted, err := s.rootManager.cryptoKeyPriv.Decrypt(acctInfo.acctKeyEncrypted)
 		if err != nil {
-			str := fmt.Sprintf("failed to decrypt private key for "+
+			return nil, err
+		}
+		return hdkeychain.NewKeyFromString(string(serializedKey))
+	}
+
+	// The wallet will only contain private keys for default accounts if the
+	// wallet's not set up as watch-only and it's been unlocked.
+	watchOnly := s.rootManager.watchOnly()
+	hasPrivateKey := !s.rootManager.isLocked() && !watchOnly
+
+	// Create the new account info with the known information. The rest of
+	// the fields are filled out below.
+	var acctInfo *accountInfo
+	switch row := rowInterface.(type) {
+	case *dbDefaultAccountRow:
+		acctInfo = &accountInfo{
+			acctName:          row.name,
+			acctKeyEncrypted:  row.privKeyEncrypted,
+			nextExternalIndex: row.nextExternalIndex,
+			nextInternalIndex: row.nextInternalIndex,
+		}
+
+		// Use the crypto public key to decrypt the account public
+		// extended key.
+		acctInfo.acctKeyPub, err = decryptKey(
+			s.rootManager.cryptoKeyPub, row.pubKeyEncrypted,
+		)
+		if err != nil {
+			str := fmt.Sprintf("failed to decrypt public key for "+
 				"account %d", account)
 			return nil, managerError(ErrCrypto, str, err)
 		}
 
-		acctKeyPriv, err := hdkeychain.NewKeyFromString(string(decrypted))
+		if hasPrivateKey {
-		if err != nil {
+			// Use the crypto private key to decrypt the account
-			str := fmt.Sprintf("failed to create extended private "+
+			// private extended keys.
-				"key for account %d", account)
+			acctInfo.acctKeyPriv, err = decryptKey(
-			return nil, managerError(ErrKeyChain, str, err)
+				s.rootManager.cryptoKeyPriv, row.privKeyEncrypted,
+			)
+			if err != nil {
+				str := fmt.Sprintf("failed to decrypt private "+
+					"key for account %d", account)
+				return nil, managerError(ErrCrypto, str, err)
+			}
 		}
-		acctInfo.acctKeyPriv = acctKeyPriv
+
+	case *dbWatchOnlyAccountRow:
+		acctInfo = &accountInfo{
+			acctName:          row.name,
+			nextExternalIndex: row.nextExternalIndex,
+			nextInternalIndex: row.nextInternalIndex,
+		}
+
+		// Use the crypto public key to decrypt the account public
+		// extended key.
+		acctInfo.acctKeyPub, err = decryptKey(
+			s.rootManager.cryptoKeyPub, row.pubKeyEncrypted,
+		)
+		if err != nil {
+			str := fmt.Sprintf("failed to decrypt public key for "+
+				"account %d", account)
+			return nil, managerError(ErrCrypto, str, err)
+		}
+
+		watchOnly = true
+		hasPrivateKey = false
+
+	default:
+		str := fmt.Sprintf("unsupported account type %T", row)
+		return nil, managerError(ErrDatabase, str, nil)
 	}
 
 	// Derive and cache the managed address for the last external address.
-	branch, index := ExternalBranch, row.nextExternalIndex
+	branch, index := ExternalBranch, acctInfo.nextExternalIndex
 	if index > 0 {
 		index--
 	}
 	lastExtAddrPath := DerivationPath{
 		InternalAccount: account,
-		Account:         acctKeyPub.ChildIndex(),
+		Account:         acctInfo.acctKeyPub.ChildIndex(),
 		Branch:          branch,
 		Index:           index,
 	}
-	lastExtKey, err := s.deriveKey(acctInfo, branch, index, private)
+	lastExtKey, err := s.deriveKey(acctInfo, branch, index, hasPrivateKey)
 	if err != nil {
 		return nil, err
 	}
@@ -384,17 +408,17 @@ func (s *ScopedKeyManager) loadAccountInfo(ns walletdb.ReadBucket,
 	acctInfo.lastExternalAddr = lastExtAddr
 
 	// Derive and cache the managed address for the last internal address.
-	branch, index = InternalBranch, row.nextInternalIndex
+	branch, index = InternalBranch, acctInfo.nextInternalIndex
 	if index > 0 {
 		index--
 	}
 	lastIntAddrPath := DerivationPath{
 		InternalAccount: account,
-		Account:         acctKeyPub.ChildIndex(),
+		Account:         acctInfo.acctKeyPub.ChildIndex(),
 		Branch:          branch,
 		Index:           index,
 	}
-	lastIntKey, err := s.deriveKey(acctInfo, branch, index, private)
+	lastIntKey, err := s.deriveKey(acctInfo, branch, index, hasPrivateKey)
 	if err != nil {
 		return nil, err
 	}
@@ -1371,7 +1395,7 @@ func (s *ScopedKeyManager) newAccount(ns walletdb.ReadWriteBucket,
 
 	// We have the encrypted account extended keys, so save them to the
 	// database
-	err = putAccountInfo(
+	err = putDefaultAccountInfo(
 		ns, &s.scope, account, acctPubEnc, acctPrivEnc, 0, 0, name,
 	)
 	if err != nil {
@@ -1435,8 +1459,9 @@ func (s *ScopedKeyManager) newAccountWatchingOnly(ns walletdb.ReadWriteBucket, a
 
 	// We have the encrypted account extended keys, so save them to the
 	// database
-	err = putAccountInfo(
+	// TODO: set master key fingerprint and addr schema.
-		ns, &s.scope, account, acctPubEnc, nil, 0, 0, name,
+	err = putWatchOnlyAccountInfo(
+		ns, &s.scope, account, acctPubEnc, 0, 0, 0, name, nil,
 	)
 	if err != nil {
 		return err
@@ -1478,29 +1503,45 @@ func (s *ScopedKeyManager) RenameAccount(ns walletdb.ReadWriteBucket,
 		return err
 	}
 
-	// Ensure the account type is a default account.
-	row, ok := rowInterface.(*dbDefaultAccountRow)
-	if !ok {
-		str := fmt.Sprintf("unsupported account type %T", row)
-		err = managerError(ErrDatabase, str, nil)
-	}
-
 	// Remove the old name key from the account id index.
 	if err = deleteAccountIDIndex(ns, &s.scope, account); err != nil {
 		return err
 	}
 
-	// Remove the old name key from the account name index.
+	switch row := rowInterface.(type) {
-	if err = deleteAccountNameIndex(ns, &s.scope, row.name); err != nil {
+	case *dbDefaultAccountRow:
-		return err
+		// Remove the old name key from the account name index.
-	}
+		if err = deleteAccountNameIndex(ns, &s.scope, row.name); err != nil {
-	err = putAccountInfo(
+			return err
-		ns, &s.scope, account, row.pubKeyEncrypted,
+		}
-		row.privKeyEncrypted, row.nextExternalIndex,
+
-		row.nextInternalIndex, name,
+		err = putDefaultAccountInfo(
-	)
+			ns, &s.scope, account, row.pubKeyEncrypted,
-	if err != nil {
+			row.privKeyEncrypted, row.nextExternalIndex,
-		return err
+			row.nextInternalIndex, name,
+		)
+		if err != nil {
+			return err
+		}
+
+	case *dbWatchOnlyAccountRow:
+		// Remove the old name key from the account name index.
+		if err = deleteAccountNameIndex(ns, &s.scope, row.name); err != nil {
+			return err
+		}
+
+		err = putWatchOnlyAccountInfo(
+			ns, &s.scope, account, row.pubKeyEncrypted,
+			row.masterKeyFingerprint, row.nextExternalIndex,
+			row.nextInternalIndex, name, row.addrSchema,
+		)
+		if err != nil {
+			return err
+		}
+
+	default:
+		str := fmt.Sprintf("unsupported account type %T", row)
+		return managerError(ErrDatabase, str, nil)
 	}
 
 	// Update in-memory account info with new name if cached and the db