Introduce new account file structure.

This changes the locations that account files (wallet.bin, utxo.bin,
and tx.bin) are searched for when opening or disk syncing accounts.
Previously, files were saved in the following layout:

  ~/.btcwallet/
    - btcwallet/
      - wallet.bin
      - tx.bin
      - utxo.bin
    - btcwallet-AccountA/
      - wallet.bin
      - tx.bin
      - utxo.bin

This format had two issues.  First, each account would require its own
directory, causing a scalability issue on unix (and perhaps other)
platforms.  Second, there was no distinction between testnet and
mainnet wallets, and if mainnet support was enabled, btcwallet would
attempt to open accounts with testnet wallets.

Instead, the following file structure is now used:

  ~/.btcwallet/
    - testnet/
      - wallet.bin
      - tx.bin
      - utxo.bin
      - AccountA-wallet.bin
      - AccountA-tx.bin
      - AccountA-utxo.bin

This solves both of the previously-mentioned issues by requiring only
two subdirectories (one each for the testnet and mainnet bitcoin
networks), and by separating the locations to open and save testnet
and mainnet account files.

At startup, a check for the old account file structure is performed.
If found, files are moved to the new locations, and the old account
directories are removed.  Account files are moved to the testnet
directory, as only testnet support is currently enabled.

The version has been bumped to 0.1.1 to reflect this change.

Fixes #16.

account.go

@@ -706,36 +706,50 @@ func (a *Account) newBlockTxOutHandler(result interface{}, e *btcjson.Error) boo
 	return false
 }
 
-// accountdir returns the directory path which holds an account's wallet, utxo,
+// accountdir returns the directory containing an account's wallet, utxo,
 // and tx files.
-func (a *Account) accountdir(cfg *config) string {
+//
-	var wname string
+// This function is deprecated and should only be used when looking up
-	if a.name == "" {
+// old (before version 0.1.1) account directories so they may be updated
-		wname = "btcwallet"
+// to the new directory structure.
+func accountdir(name string, cfg *config) string {
+	var adir string
+	if name == "" { // default account
+		adir = "btcwallet"
 	} else {
-		wname = fmt.Sprintf("btcwallet-%s", a.name)
+		adir = fmt.Sprintf("btcwallet-%s", name)
 	}
 
-	return filepath.Join(cfg.DataDir, wname)
+	return filepath.Join(cfg.DataDir, adir)
 }
 
-// checkCreateAccountDir checks that path exists and is a directory.
+func networkDir(net btcwire.BitcoinNet) string {
+	var netname string
+	if net == btcwire.MainNet {
+		netname = "mainnet"
+	} else {
+		netname = "testnet"
+	}
+	return filepath.Join(cfg.DataDir, netname)
+}
+
+// checkCreateDir checks that the path exists and is a directory.
 // If path does not exist, it is created.
-func (a *Account) checkCreateAccountDir(path string) error {
+func checkCreateDir(path string) error {
-	fi, err := os.Stat(path)
+	if fi, err := os.Stat(path); err != nil {
-	if err != nil {
 		if os.IsNotExist(err) {
 			// Attempt data directory creation
 			if err = os.MkdirAll(path, 0700); err != nil {
-				return fmt.Errorf("cannot create account directory: %s", err)
+				return fmt.Errorf("cannot create network directory: %s", err)
 			}
 		} else {
-			return fmt.Errorf("error checking account directory: %s", err)
+			return fmt.Errorf("error checking network directory: %s", err)
 		}
 	} else {
 		if !fi.IsDir() {
-			return fmt.Errorf("path '%s' is not a directory", cfg.DataDir)
+			return fmt.Errorf("path '%s' is not a directory", path)
 		}
 	}
+
 	return nil
 }

accountstore.go

@@ -24,7 +24,6 @@ import (
 	"github.com/conformal/btcwallet/wallet"
 	"github.com/conformal/btcwire"
 	"os"
-	"path/filepath"
 	"sync"
 )
 
@@ -346,20 +345,19 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error {
 
 	wlt := new(wallet.Wallet)
 
-	account := &Account{
+	a := &Account{
 		Wallet: wlt,
 		name:   name,
 	}
 
-	var finalErr error
+	netdir := networkDir(cfg.Net())
-	adir := account.accountdir(cfg)
+	if err := checkCreateDir(netdir); err != nil {
-	if err := account.checkCreateAccountDir(adir); err != nil {
 		return err
 	}
 
-	wfilepath := filepath.Join(adir, "wallet.bin")
+	wfilepath := accountFilename("wallet.bin", name, netdir)
-	utxofilepath := filepath.Join(adir, "utxo.bin")
+	utxofilepath := accountFilename("utxo.bin", name, netdir)
-	txfilepath := filepath.Join(adir, "tx.bin")
+	txfilepath := accountFilename("tx.bin", name, netdir)
 	var wfile, utxofile, txfile *os.File
 
 	// Read wallet file.
@@ -379,6 +377,7 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error {
 
 	// Read tx file.  If this fails, return a ErrNoTxs error and let
 	// the caller decide if a rescan is necessary.
+	var finalErr error
 	if txfile, err = os.Open(txfilepath); err != nil {
 		log.Errorf("cannot open tx file: %s", err)
 		// This is not a error we should immediately return with,
@@ -392,7 +391,7 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error {
 			log.Errorf("cannot read tx file: %s", err)
 			finalErr = ErrNoTxs
 		} else {
-			account.TxStore.s = txs
+			a.TxStore.s = txs
 		}
 	}
 
@@ -409,7 +408,7 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error {
 			log.Errorf("cannot read utxo file: %s", err)
 			finalErr = ErrNoUtxos
 		} else {
-			account.UtxoStore.s = utxos
+			a.UtxoStore.s = utxos
 		}
 	}
 
@@ -417,16 +416,16 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error {
 	case ErrNoTxs:
 		// Do nothing special for now.  This will be implemented when
 		// the tx history file is properly written.
-		store.accounts[name] = account
+		store.accounts[name] = a
 
 	case ErrNoUtxos:
 		// Add wallet, but mark wallet as needing a full rescan since
 		// the wallet creation block.  This will take place when btcd
 		// connects.
-		account.fullRescan = true
+		a.fullRescan = true
-		store.accounts[name] = account
+		store.accounts[name] = a
 	case nil:
-		store.accounts[name] = account
+		store.accounts[name] = a
 
 	default:
 		log.Warnf("cannot open wallet: %v", err)

cmd.go

@@ -204,7 +204,11 @@ func main() {
 		}()
 	}
 
-	// Open default account
+	// Check and update any old file locations.
+	updateOldFileLocations()
+
+	// Open default account.
+	// TODO(jrick): open all available accounts.
 	err = accountstore.OpenAccount("", cfg)
 	if err != nil {
 		log.Warnf("cannot open default account: %v", err)

config.go

@@ -30,7 +30,6 @@ import (
 const (
 	defaultCAFilename     = "btcd.cert"
 	defaultConfigFilename = "btcwallet.conf"
-	defaultDataDirname    = "data"
 	defaultBtcNet         = btcwire.TestNet3
 	defaultLogLevel       = "info"
 )
@@ -217,6 +216,13 @@ func loadConfig() (*config, []string, error) {
 	return &cfg, remainingArgs, nil
 }
 
+func (c *config) Net() btcwire.BitcoinNet {
+	if cfg.MainNet {
+		return btcwire.MainNet
+	}
+	return btcwire.TestNet3
+}
+
 // validLogLevel returns whether or not logLevel is a valid debug log level.
 func validLogLevel(logLevel string) bool {
 	switch logLevel {

disksync.go

@@ -34,6 +34,19 @@ var (
 	}
 )
 
+// accountFilename returns the filepath of an account file given the
+// filename suffix ("wallet.bin", "tx.bin", or "utxo.bin"), account
+// name and the network directory holding the file.
+func accountFilename(suffix, account, netdir string) string {
+	if account == "" {
+		// default account
+		return filepath.Join(netdir, suffix)
+	}
+
+	// non-default account
+	return filepath.Join(netdir, fmt.Sprintf("%v-%v", account, suffix))
+}
+
 // DirtyAccountSyncer synces dirty accounts for cases where the updated
 // information was not required to be immediately written to disk.  Accounts
 // may be added to dirtyAccounts and will be checked and processed every 10
@@ -63,20 +76,20 @@ func DirtyAccountSyncer() {
 
 // writeDirtyToDisk checks for the dirty flag on an account's wallet,
 // txstore, and utxostore, writing them to disk if any are dirty.
-func (w *Account) writeDirtyToDisk() error {
+func (a *Account) writeDirtyToDisk() error {
 	// Temporary files append the current time to the normal file name.
 	// In caes of failure, the most recent temporary file can be inspected
 	// for validity, and moved to replace the main file.
 	timeStr := fmt.Sprintf("%v", time.Now().Unix())
 
-	adir := w.accountdir(cfg)
+	netdir := networkDir(cfg.Net())
-	if err := w.checkCreateAccountDir(adir); err != nil {
+	if err := checkCreateDir(netdir); err != nil {
 		return err
 	}
 
-	wfilepath := filepath.Join(adir, "wallet.bin")
+	wfilepath := accountFilename("wallet.bin", a.name, netdir)
-	txfilepath := filepath.Join(adir, "tx.bin")
+	txfilepath := accountFilename("tx.bin", a.name, netdir)
-	utxofilepath := filepath.Join(adir, "utxo.bin")
+	utxofilepath := accountFilename("utxo.bin", a.name, netdir)
 
 	// UTXOs and transactions are synced to disk first.  This prevents
 	// any races from saving a wallet marked to be synced with block N
@@ -84,18 +97,18 @@ func (w *Account) writeDirtyToDisk() error {
 	// with block N-1.
 
 	// UTXOs
-	w.UtxoStore.RLock()
+	a.UtxoStore.RLock()
-	dirty := w.TxStore.dirty
+	dirty := a.TxStore.dirty
-	w.UtxoStore.RUnlock()
+	a.UtxoStore.RUnlock()
 	if dirty {
-		w.UtxoStore.Lock()
+		a.UtxoStore.Lock()
-		defer w.UtxoStore.Unlock()
+		defer a.UtxoStore.Unlock()
 		tmpfilepath := utxofilepath + "-" + timeStr
 		tmpfile, err := os.Create(tmpfilepath)
 		if err != nil {
 			return err
 		}
-		if _, err = w.UtxoStore.s.WriteTo(tmpfile); err != nil {
+		if _, err = a.UtxoStore.s.WriteTo(tmpfile); err != nil {
 			return err
 		}
 		tmpfile.Close()
@@ -106,22 +119,22 @@ func (w *Account) writeDirtyToDisk() error {
 			return err
 		}
 
-		w.UtxoStore.dirty = false
+		a.UtxoStore.dirty = false
 	}
 
 	// Transactions
-	w.TxStore.RLock()
+	a.TxStore.RLock()
-	dirty = w.TxStore.dirty
+	dirty = a.TxStore.dirty
-	w.TxStore.RUnlock()
+	a.TxStore.RUnlock()
 	if dirty {
-		w.TxStore.Lock()
+		a.TxStore.Lock()
-		defer w.TxStore.Unlock()
+		defer a.TxStore.Unlock()
 		tmpfilepath := txfilepath + "-" + timeStr
 		tmpfile, err := os.Create(tmpfilepath)
 		if err != nil {
 			return err
 		}
-		if _, err = w.TxStore.s.WriteTo(tmpfile); err != nil {
+		if _, err = a.TxStore.s.WriteTo(tmpfile); err != nil {
 			return err
 		}
 		tmpfile.Close()
@@ -132,22 +145,22 @@ func (w *Account) writeDirtyToDisk() error {
 			return err
 		}
 
-		w.TxStore.dirty = false
+		a.TxStore.dirty = false
 	}
 
 	// Wallet
-	w.mtx.RLock()
+	a.mtx.RLock()
-	dirty = w.dirty
+	dirty = a.dirty
-	w.mtx.RUnlock()
+	a.mtx.RUnlock()
 	if dirty {
-		w.mtx.Lock()
+		a.mtx.Lock()
-		defer w.mtx.Unlock()
+		defer a.mtx.Unlock()
 		tmpfilepath := wfilepath + "-" + timeStr
 		tmpfile, err := os.Create(tmpfilepath)
 		if err != nil {
 			return err
 		}
-		if _, err = w.WriteTo(tmpfile); err != nil {
+		if _, err = a.WriteTo(tmpfile); err != nil {
 			return err
 		}
 		tmpfile.Close()
@@ -158,7 +171,7 @@ func (w *Account) writeDirtyToDisk() error {
 			return err
 		}
 
-		w.dirty = false
+		a.dirty = false
 	}
 
 	return nil

sockets.go

@@ -810,14 +810,7 @@ func BtcdHandshake(ws *websocket.Conn) error {
 			return true
 		}
 
-		var walletNetwork btcwire.BitcoinNet
+		correctNetwork <- btcwire.BitcoinNet(fnet) == cfg.Net()
-		if cfg.MainNet {
-			walletNetwork = btcwire.MainNet
-		} else {
-			walletNetwork = btcwire.TestNet3
-		}
-
-		correctNetwork <- btcwire.BitcoinNet(fnet) == walletNetwork
 
 		// No additional replies expected, remove handler.
 		return true

updates.go

@@ -0,0 +1,162 @@
+/*
+ * Copyright (c) 2013 Conformal Systems LLC <info@conformal.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ErrNotAccountDir describes an error where a directory in the btcwallet
+// data directory cannot be parsed as a directory holding account files.
+var ErrNotAccountDir = errors.New("directory is not an account directory")
+
+// updateOldFileLocations moves files for wallets, transactions, and
+// recorded unspent transaction outputs to more recent locations.
+//
+// If any errors are encounted during this function, the application is
+// closed.
+func updateOldFileLocations() {
+	// Before version 0.1.1, accounts were saved with the following
+	// format:
+	//
+	//   ~/.btcwallet/
+	//     - btcwallet/
+	//       - wallet.bin
+	//       - tx.bin
+	//       - utxo.bin
+	//     - btcwallet-AccountA/
+	//       - wallet.bin
+	//       - tx.bin
+	//       - utxo.bin
+	//
+	// This format does not scale well (see Github issue #16), and
+	// since version 0.1.1, the above directory format has changed
+	// to the following:
+	//
+	//   ~/.btcwallet/
+	//     - testnet/
+	//       - wallet.bin
+	//       - tx.bin
+	//       - utxo.bin
+	//       - AccountA-wallet.bin
+	//       - AccountA-tx.bin
+	//       - AccountA-utxo.bin
+	//
+	// Previous account files are placed in the testnet directory
+	// as 0.1.0 and earlier only ran on testnet.
+
+	datafi, err := os.Open(cfg.DataDir)
+	if err != nil {
+		return
+	}
+	defer datafi.Close()
+
+	// Get info on all files in the data directory.
+	fi, err := datafi.Readdir(0)
+	if err != nil {
+		log.Errorf("Cannot read files in data directory: %v", err)
+		os.Exit(1)
+	}
+
+	// Create testnet directory, if it doesn't already exist.
+	netdir := filepath.Join(cfg.DataDir, "testnet")
+	if err := checkCreateDir(netdir); err != nil {
+		log.Errorf("Cannot continue without a testnet directory: %v", err)
+		os.Exit(1)
+	}
+
+	// Check all files in the datadir for old accounts to update.
+	for i := range fi {
+		// Ignore non-directories.
+		if !fi[i].IsDir() {
+			continue
+		}
+
+		account, err := parseOldAccountDir(cfg.DataDir, fi[i].Name())
+		switch err {
+		case nil:
+			break
+
+		case ErrNotAccountDir:
+			continue
+
+		default: // all other non-nil errors
+			log.Errorf("Cannot open old account directory: %v", err)
+			os.Exit(1)
+		}
+
+		log.Infof("Updating old file locations for account %v\n", account)
+
+		// Move old wallet.bin, if any.
+		old := filepath.Join(cfg.DataDir, fi[i].Name(), "wallet.bin")
+		if fileExists(old) {
+			new := accountFilename("wallet.bin", account, netdir)
+			if err := os.Rename(old, new); err != nil {
+				log.Errorf("Cannot move old %v for account %v to new location: %v",
+					"wallet.bin", account, err)
+				os.Exit(1)
+			}
+		}
+
+		// Move old tx.bin, if any.
+		old = filepath.Join(cfg.DataDir, fi[i].Name(), "tx.bin")
+		if fileExists(old) {
+			new := accountFilename("tx.bin", account, netdir)
+			if err := os.Rename(old, new); err != nil {
+				log.Errorf("Cannot move old %v for account %v to new location: %v",
+					"tx.bin", account, err)
+				os.Exit(1)
+			}
+		}
+
+		// Move old utxo.bin, if any.
+		old = filepath.Join(cfg.DataDir, fi[i].Name(), "utxo.bin")
+		if fileExists(old) {
+			new := accountFilename("utxo.bin", account, netdir)
+			if err := os.Rename(old, new); err != nil {
+				log.Errorf("Cannot move old %v for account %v to new location: %v",
+					"utxo.bin", account, err)
+				os.Exit(1)
+			}
+		}
+
+		// Cleanup old account directory.
+		os.RemoveAll(filepath.Join(cfg.DataDir, fi[i].Name()))
+	}
+}
+
+type oldAccountDir struct {
+	account string
+	dir     *os.File
+}
+
+func parseOldAccountDir(dir, base string) (string, error) {
+	if base == "btcwallet" {
+		return "", nil
+	}
+
+	const accountPrefix = "btcwallet-"
+	if strings.HasPrefix(base, accountPrefix) {
+		account := strings.TrimPrefix(base, accountPrefix)
+		return account, nil
+	}
+
+	return "", ErrNotAccountDir
+}

version.go

@@ -30,7 +30,7 @@ const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqr
 const (
 	appMajor uint = 0
 	appMinor uint = 1
-	appPatch uint = 0
+	appPatch uint = 1
 
 	// appPreRelease MUST only contain characters from semanticAlphabet
 	// per the semantic versioning spec.