From ce23523ed72e57b9dafa315b446c433e7b533558 Mon Sep 17 00:00:00 2001 From: Josh Rickmar Date: Wed, 4 Dec 2013 20:16:50 -0500 Subject: [PATCH] Introduce new account file structure. This changes the locations that account files (wallet.bin, utxo.bin, and tx.bin) are searched for when opening or disk syncing accounts. Previously, files were saved in the following layout: ~/.btcwallet/ - btcwallet/ - wallet.bin - tx.bin - utxo.bin - btcwallet-AccountA/ - wallet.bin - tx.bin - utxo.bin This format had two issues. First, each account would require its own directory, causing a scalability issue on unix (and perhaps other) platforms. Second, there was no distinction between testnet and mainnet wallets, and if mainnet support was enabled, btcwallet would attempt to open accounts with testnet wallets. Instead, the following file structure is now used: ~/.btcwallet/ - testnet/ - wallet.bin - tx.bin - utxo.bin - AccountA-wallet.bin - AccountA-tx.bin - AccountA-utxo.bin This solves both of the previously-mentioned issues by requiring only two subdirectories (one each for the testnet and mainnet bitcoin networks), and by separating the locations to open and save testnet and mainnet account files. At startup, a check for the old account file structure is performed. If found, files are moved to the new locations, and the old account directories are removed. Account files are moved to the testnet directory, as only testnet support is currently enabled. The version has been bumped to 0.1.1 to reflect this change. Fixes #16. --- account.go | 42 ++++++++----- accountstore.go | 27 ++++---- cmd.go | 6 +- config.go | 8 ++- disksync.go | 67 ++++++++++++-------- sockets.go | 9 +-- updates.go | 162 ++++++++++++++++++++++++++++++++++++++++++++++++ version.go | 2 +- 8 files changed, 257 insertions(+), 66 deletions(-) create mode 100644 updates.go diff --git a/account.go b/account.go index bfb2e6a..abad32a 100644 --- a/account.go +++ b/account.go @@ -706,36 +706,50 @@ func (a *Account) newBlockTxOutHandler(result interface{}, e *btcjson.Error) boo return false } -// accountdir returns the directory path which holds an account's wallet, utxo, +// accountdir returns the directory containing an account's wallet, utxo, // and tx files. -func (a *Account) accountdir(cfg *config) string { - var wname string - if a.name == "" { - wname = "btcwallet" +// +// This function is deprecated and should only be used when looking up +// old (before version 0.1.1) account directories so they may be updated +// to the new directory structure. +func accountdir(name string, cfg *config) string { + var adir string + if name == "" { // default account + adir = "btcwallet" } else { - wname = fmt.Sprintf("btcwallet-%s", a.name) + adir = fmt.Sprintf("btcwallet-%s", name) } - return filepath.Join(cfg.DataDir, wname) + return filepath.Join(cfg.DataDir, adir) } -// checkCreateAccountDir checks that path exists and is a directory. +func networkDir(net btcwire.BitcoinNet) string { + var netname string + if net == btcwire.MainNet { + netname = "mainnet" + } else { + netname = "testnet" + } + return filepath.Join(cfg.DataDir, netname) +} + +// checkCreateDir checks that the path exists and is a directory. // If path does not exist, it is created. -func (a *Account) checkCreateAccountDir(path string) error { - fi, err := os.Stat(path) - if err != nil { +func checkCreateDir(path string) error { + if fi, err := os.Stat(path); err != nil { if os.IsNotExist(err) { // Attempt data directory creation if err = os.MkdirAll(path, 0700); err != nil { - return fmt.Errorf("cannot create account directory: %s", err) + return fmt.Errorf("cannot create network directory: %s", err) } } else { - return fmt.Errorf("error checking account directory: %s", err) + return fmt.Errorf("error checking network directory: %s", err) } } else { if !fi.IsDir() { - return fmt.Errorf("path '%s' is not a directory", cfg.DataDir) + return fmt.Errorf("path '%s' is not a directory", path) } } + return nil } diff --git a/accountstore.go b/accountstore.go index 6fd4d22..8fa6e57 100644 --- a/accountstore.go +++ b/accountstore.go @@ -24,7 +24,6 @@ import ( "github.com/conformal/btcwallet/wallet" "github.com/conformal/btcwire" "os" - "path/filepath" "sync" ) @@ -346,20 +345,19 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error { wlt := new(wallet.Wallet) - account := &Account{ + a := &Account{ Wallet: wlt, name: name, } - var finalErr error - adir := account.accountdir(cfg) - if err := account.checkCreateAccountDir(adir); err != nil { + netdir := networkDir(cfg.Net()) + if err := checkCreateDir(netdir); err != nil { return err } - wfilepath := filepath.Join(adir, "wallet.bin") - utxofilepath := filepath.Join(adir, "utxo.bin") - txfilepath := filepath.Join(adir, "tx.bin") + wfilepath := accountFilename("wallet.bin", name, netdir) + utxofilepath := accountFilename("utxo.bin", name, netdir) + txfilepath := accountFilename("tx.bin", name, netdir) var wfile, utxofile, txfile *os.File // Read wallet file. @@ -379,6 +377,7 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error { // Read tx file. If this fails, return a ErrNoTxs error and let // the caller decide if a rescan is necessary. + var finalErr error if txfile, err = os.Open(txfilepath); err != nil { log.Errorf("cannot open tx file: %s", err) // This is not a error we should immediately return with, @@ -392,7 +391,7 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error { log.Errorf("cannot read tx file: %s", err) finalErr = ErrNoTxs } else { - account.TxStore.s = txs + a.TxStore.s = txs } } @@ -409,7 +408,7 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error { log.Errorf("cannot read utxo file: %s", err) finalErr = ErrNoUtxos } else { - account.UtxoStore.s = utxos + a.UtxoStore.s = utxos } } @@ -417,16 +416,16 @@ func (store *AccountStore) OpenAccount(name string, cfg *config) error { case ErrNoTxs: // Do nothing special for now. This will be implemented when // the tx history file is properly written. - store.accounts[name] = account + store.accounts[name] = a case ErrNoUtxos: // Add wallet, but mark wallet as needing a full rescan since // the wallet creation block. This will take place when btcd // connects. - account.fullRescan = true - store.accounts[name] = account + a.fullRescan = true + store.accounts[name] = a case nil: - store.accounts[name] = account + store.accounts[name] = a default: log.Warnf("cannot open wallet: %v", err) diff --git a/cmd.go b/cmd.go index de03fa7..1dd2299 100644 --- a/cmd.go +++ b/cmd.go @@ -204,7 +204,11 @@ func main() { }() } - // Open default account + // Check and update any old file locations. + updateOldFileLocations() + + // Open default account. + // TODO(jrick): open all available accounts. err = accountstore.OpenAccount("", cfg) if err != nil { log.Warnf("cannot open default account: %v", err) diff --git a/config.go b/config.go index a41791f..abb2bb8 100644 --- a/config.go +++ b/config.go @@ -30,7 +30,6 @@ import ( const ( defaultCAFilename = "btcd.cert" defaultConfigFilename = "btcwallet.conf" - defaultDataDirname = "data" defaultBtcNet = btcwire.TestNet3 defaultLogLevel = "info" ) @@ -217,6 +216,13 @@ func loadConfig() (*config, []string, error) { return &cfg, remainingArgs, nil } +func (c *config) Net() btcwire.BitcoinNet { + if cfg.MainNet { + return btcwire.MainNet + } + return btcwire.TestNet3 +} + // validLogLevel returns whether or not logLevel is a valid debug log level. func validLogLevel(logLevel string) bool { switch logLevel { diff --git a/disksync.go b/disksync.go index 2a529b8..d00e6eb 100644 --- a/disksync.go +++ b/disksync.go @@ -34,6 +34,19 @@ var ( } ) +// accountFilename returns the filepath of an account file given the +// filename suffix ("wallet.bin", "tx.bin", or "utxo.bin"), account +// name and the network directory holding the file. +func accountFilename(suffix, account, netdir string) string { + if account == "" { + // default account + return filepath.Join(netdir, suffix) + } + + // non-default account + return filepath.Join(netdir, fmt.Sprintf("%v-%v", account, suffix)) +} + // DirtyAccountSyncer synces dirty accounts for cases where the updated // information was not required to be immediately written to disk. Accounts // may be added to dirtyAccounts and will be checked and processed every 10 @@ -63,20 +76,20 @@ func DirtyAccountSyncer() { // writeDirtyToDisk checks for the dirty flag on an account's wallet, // txstore, and utxostore, writing them to disk if any are dirty. -func (w *Account) writeDirtyToDisk() error { +func (a *Account) writeDirtyToDisk() error { // Temporary files append the current time to the normal file name. // In caes of failure, the most recent temporary file can be inspected // for validity, and moved to replace the main file. timeStr := fmt.Sprintf("%v", time.Now().Unix()) - adir := w.accountdir(cfg) - if err := w.checkCreateAccountDir(adir); err != nil { + netdir := networkDir(cfg.Net()) + if err := checkCreateDir(netdir); err != nil { return err } - wfilepath := filepath.Join(adir, "wallet.bin") - txfilepath := filepath.Join(adir, "tx.bin") - utxofilepath := filepath.Join(adir, "utxo.bin") + wfilepath := accountFilename("wallet.bin", a.name, netdir) + txfilepath := accountFilename("tx.bin", a.name, netdir) + utxofilepath := accountFilename("utxo.bin", a.name, netdir) // UTXOs and transactions are synced to disk first. This prevents // any races from saving a wallet marked to be synced with block N @@ -84,18 +97,18 @@ func (w *Account) writeDirtyToDisk() error { // with block N-1. // UTXOs - w.UtxoStore.RLock() - dirty := w.TxStore.dirty - w.UtxoStore.RUnlock() + a.UtxoStore.RLock() + dirty := a.TxStore.dirty + a.UtxoStore.RUnlock() if dirty { - w.UtxoStore.Lock() - defer w.UtxoStore.Unlock() + a.UtxoStore.Lock() + defer a.UtxoStore.Unlock() tmpfilepath := utxofilepath + "-" + timeStr tmpfile, err := os.Create(tmpfilepath) if err != nil { return err } - if _, err = w.UtxoStore.s.WriteTo(tmpfile); err != nil { + if _, err = a.UtxoStore.s.WriteTo(tmpfile); err != nil { return err } tmpfile.Close() @@ -106,22 +119,22 @@ func (w *Account) writeDirtyToDisk() error { return err } - w.UtxoStore.dirty = false + a.UtxoStore.dirty = false } // Transactions - w.TxStore.RLock() - dirty = w.TxStore.dirty - w.TxStore.RUnlock() + a.TxStore.RLock() + dirty = a.TxStore.dirty + a.TxStore.RUnlock() if dirty { - w.TxStore.Lock() - defer w.TxStore.Unlock() + a.TxStore.Lock() + defer a.TxStore.Unlock() tmpfilepath := txfilepath + "-" + timeStr tmpfile, err := os.Create(tmpfilepath) if err != nil { return err } - if _, err = w.TxStore.s.WriteTo(tmpfile); err != nil { + if _, err = a.TxStore.s.WriteTo(tmpfile); err != nil { return err } tmpfile.Close() @@ -132,22 +145,22 @@ func (w *Account) writeDirtyToDisk() error { return err } - w.TxStore.dirty = false + a.TxStore.dirty = false } // Wallet - w.mtx.RLock() - dirty = w.dirty - w.mtx.RUnlock() + a.mtx.RLock() + dirty = a.dirty + a.mtx.RUnlock() if dirty { - w.mtx.Lock() - defer w.mtx.Unlock() + a.mtx.Lock() + defer a.mtx.Unlock() tmpfilepath := wfilepath + "-" + timeStr tmpfile, err := os.Create(tmpfilepath) if err != nil { return err } - if _, err = w.WriteTo(tmpfile); err != nil { + if _, err = a.WriteTo(tmpfile); err != nil { return err } tmpfile.Close() @@ -158,7 +171,7 @@ func (w *Account) writeDirtyToDisk() error { return err } - w.dirty = false + a.dirty = false } return nil diff --git a/sockets.go b/sockets.go index 459b6e2..1b17112 100644 --- a/sockets.go +++ b/sockets.go @@ -810,14 +810,7 @@ func BtcdHandshake(ws *websocket.Conn) error { return true } - var walletNetwork btcwire.BitcoinNet - if cfg.MainNet { - walletNetwork = btcwire.MainNet - } else { - walletNetwork = btcwire.TestNet3 - } - - correctNetwork <- btcwire.BitcoinNet(fnet) == walletNetwork + correctNetwork <- btcwire.BitcoinNet(fnet) == cfg.Net() // No additional replies expected, remove handler. return true diff --git a/updates.go b/updates.go new file mode 100644 index 0000000..ae0e8a9 --- /dev/null +++ b/updates.go @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2013 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package main + +import ( + "errors" + "os" + "path/filepath" + "strings" +) + +// ErrNotAccountDir describes an error where a directory in the btcwallet +// data directory cannot be parsed as a directory holding account files. +var ErrNotAccountDir = errors.New("directory is not an account directory") + +// updateOldFileLocations moves files for wallets, transactions, and +// recorded unspent transaction outputs to more recent locations. +// +// If any errors are encounted during this function, the application is +// closed. +func updateOldFileLocations() { + // Before version 0.1.1, accounts were saved with the following + // format: + // + // ~/.btcwallet/ + // - btcwallet/ + // - wallet.bin + // - tx.bin + // - utxo.bin + // - btcwallet-AccountA/ + // - wallet.bin + // - tx.bin + // - utxo.bin + // + // This format does not scale well (see Github issue #16), and + // since version 0.1.1, the above directory format has changed + // to the following: + // + // ~/.btcwallet/ + // - testnet/ + // - wallet.bin + // - tx.bin + // - utxo.bin + // - AccountA-wallet.bin + // - AccountA-tx.bin + // - AccountA-utxo.bin + // + // Previous account files are placed in the testnet directory + // as 0.1.0 and earlier only ran on testnet. + + datafi, err := os.Open(cfg.DataDir) + if err != nil { + return + } + defer datafi.Close() + + // Get info on all files in the data directory. + fi, err := datafi.Readdir(0) + if err != nil { + log.Errorf("Cannot read files in data directory: %v", err) + os.Exit(1) + } + + // Create testnet directory, if it doesn't already exist. + netdir := filepath.Join(cfg.DataDir, "testnet") + if err := checkCreateDir(netdir); err != nil { + log.Errorf("Cannot continue without a testnet directory: %v", err) + os.Exit(1) + } + + // Check all files in the datadir for old accounts to update. + for i := range fi { + // Ignore non-directories. + if !fi[i].IsDir() { + continue + } + + account, err := parseOldAccountDir(cfg.DataDir, fi[i].Name()) + switch err { + case nil: + break + + case ErrNotAccountDir: + continue + + default: // all other non-nil errors + log.Errorf("Cannot open old account directory: %v", err) + os.Exit(1) + } + + log.Infof("Updating old file locations for account %v\n", account) + + // Move old wallet.bin, if any. + old := filepath.Join(cfg.DataDir, fi[i].Name(), "wallet.bin") + if fileExists(old) { + new := accountFilename("wallet.bin", account, netdir) + if err := os.Rename(old, new); err != nil { + log.Errorf("Cannot move old %v for account %v to new location: %v", + "wallet.bin", account, err) + os.Exit(1) + } + } + + // Move old tx.bin, if any. + old = filepath.Join(cfg.DataDir, fi[i].Name(), "tx.bin") + if fileExists(old) { + new := accountFilename("tx.bin", account, netdir) + if err := os.Rename(old, new); err != nil { + log.Errorf("Cannot move old %v for account %v to new location: %v", + "tx.bin", account, err) + os.Exit(1) + } + } + + // Move old utxo.bin, if any. + old = filepath.Join(cfg.DataDir, fi[i].Name(), "utxo.bin") + if fileExists(old) { + new := accountFilename("utxo.bin", account, netdir) + if err := os.Rename(old, new); err != nil { + log.Errorf("Cannot move old %v for account %v to new location: %v", + "utxo.bin", account, err) + os.Exit(1) + } + } + + // Cleanup old account directory. + os.RemoveAll(filepath.Join(cfg.DataDir, fi[i].Name())) + } +} + +type oldAccountDir struct { + account string + dir *os.File +} + +func parseOldAccountDir(dir, base string) (string, error) { + if base == "btcwallet" { + return "", nil + } + + const accountPrefix = "btcwallet-" + if strings.HasPrefix(base, accountPrefix) { + account := strings.TrimPrefix(base, accountPrefix) + return account, nil + } + + return "", ErrNotAccountDir +} diff --git a/version.go b/version.go index cf6ec3f..cdae29b 100644 --- a/version.go +++ b/version.go @@ -30,7 +30,7 @@ const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqr const ( appMajor uint = 0 appMinor uint = 1 - appPatch uint = 0 + appPatch uint = 1 // appPreRelease MUST only contain characters from semanticAlphabet // per the semantic versioning spec.