diff --git a/cmd.go b/cmd.go index d4af1a3..4bdb70a 100644 --- a/cmd.go +++ b/cmd.go @@ -32,23 +32,32 @@ import ( "time" ) -const ( - satoshiPerBTC = 100000000 -) - var ( // ErrNoWallet describes an error where a wallet does not exist and // must be created first. ErrNoWallet = errors.New("wallet file does not exist") + // ErrNoUtxos describes an error where the wallet file was successfully + // read, but the UTXO file was not. To properly handle this error, + // a rescan should be done since the wallet creation block. + ErrNoUtxos = errors.New("utxo file cannot be read") + + // ErrNoTxs describes an error where the wallet and UTXO files were + // successfully read, but the TX history file was not. It is up to + // the caller whether this necessitates a rescan or not. + ErrNoTxs = errors.New("tx file cannot be read") + cfg *config - curHeight = struct { + curBlock = struct { sync.RWMutex - h int64 + wallet.BlockStamp }{ - h: btcutil.BlockHeightUnknown, + BlockStamp: wallet.BlockStamp{ + Height: int32(btcutil.BlockHeightUnknown), + }, } + wallets = NewBtcWalletStore() ) @@ -61,6 +70,7 @@ type BtcWallet struct { mtx sync.RWMutex name string dirty bool + fullRescan bool NewBlockTxSeqN uint64 SpentOutpointSeqN uint64 UtxoStore struct { @@ -95,7 +105,7 @@ func NewBtcWalletStore() *BtcWalletStore { // // TODO(jrick): This must also roll back the UTXO and TX stores, and notify // all wallets of new account balances. -func (s *BtcWalletStore) Rollback(height int64, hash *btcwire.ShaHash) { +func (s *BtcWalletStore) Rollback(height int32, hash *btcwire.ShaHash) { for _, w := range s.m { w.Rollback(height, hash) } @@ -105,7 +115,7 @@ func (s *BtcWalletStore) Rollback(height int64, hash *btcwire.ShaHash) { // with the passed chainheight and block hash was connected to the main // chain. This is used to remove transactions and utxos for each wallet // that occured on a chain no longer considered to be the main chain. -func (w *BtcWallet) Rollback(height int64, hash *btcwire.ShaHash) { +func (w *BtcWallet) Rollback(height int32, hash *btcwire.ShaHash) { w.UtxoStore.Lock() w.UtxoStore.dirty = w.UtxoStore.dirty || w.UtxoStore.s.Rollback(height, hash) w.UtxoStore.Unlock() @@ -157,9 +167,11 @@ func OpenWallet(cfg *config, account string) (*BtcWallet, error) { } wfilepath := filepath.Join(wdir, "wallet.bin") - txfilepath := filepath.Join(wdir, "tx.bin") utxofilepath := filepath.Join(wdir, "utxo.bin") - var wfile, txfile, utxofile *os.File + txfilepath := filepath.Join(wdir, "tx.bin") + var wfile, utxofile, txfile *os.File + + // Read wallet file. if wfile, err = os.Open(wfilepath); err != nil { if os.IsNotExist(err) { // Must create and save wallet first. @@ -168,80 +180,114 @@ func OpenWallet(cfg *config, account string) (*BtcWallet, error) { return nil, fmt.Errorf("cannot open wallet file: %s", err) } defer wfile.Close() - if txfile, err = os.Open(txfilepath); err != nil { - if os.IsNotExist(err) { - if txfile, err = os.Create(txfilepath); err != nil { - return nil, fmt.Errorf("cannot create tx file: %s", err) - } - } else { - return nil, fmt.Errorf("cannot open tx file: %s", err) - } - } - defer txfile.Close() - if utxofile, err = os.Open(utxofilepath); err != nil { - if os.IsNotExist(err) { - if utxofile, err = os.Create(utxofilepath); err != nil { - return nil, fmt.Errorf("cannot create utxo file: %s", err) - } - } else { - return nil, fmt.Errorf("cannot open utxo file: %s", err) - } - } - defer utxofile.Close() wlt := new(wallet.Wallet) if _, err = wlt.ReadFrom(wfile); err != nil { return nil, fmt.Errorf("cannot read wallet: %s", err) } - var txs tx.TxStore - if _, err = txs.ReadFrom(txfile); err != nil { - return nil, fmt.Errorf("cannot read tx file: %s", err) - } - - var utxos tx.UtxoStore - if _, err = utxos.ReadFrom(utxofile); err != nil { - return nil, fmt.Errorf("cannot read utxo file: %s", err) - } - w := &BtcWallet{ Wallet: wlt, name: account, } + + // Read utxo file. If this fails, return a ErrNoUtxos error so a + // rescan can be done since the wallet creation block. + var utxos tx.UtxoStore + if utxofile, err = os.Open(utxofilepath); err != nil { + log.Errorf("cannot open utxo file: %s", err) + return w, ErrNoUtxos + } + defer utxofile.Close() + if _, err = utxos.ReadFrom(utxofile); err != nil { + log.Errorf("cannot read utxo file: %s", err) + return w, ErrNoUtxos + } w.UtxoStore.s = utxos + + // Read tx file. If this fails, return a ErrNoTxs error and let + // the caller decide if a rescan is necessary. + if txfile, err = os.Open(txfilepath); err != nil { + log.Errorf("cannot open tx file: %s", err) + return w, ErrNoTxs + } + defer txfile.Close() + var txs tx.TxStore + if _, err = txs.ReadFrom(txfile); err != nil { + log.Errorf("cannot read tx file: %s", err) + return w, ErrNoTxs + } w.TxStore.s = txs return w, nil } -func getCurHeight() (height int64) { - curHeight.RLock() - height = curHeight.h - curHeight.RUnlock() - if height != btcutil.BlockHeightUnknown { - return height +// GetCurBlock returns the blockchain height and SHA hash of the most +// recently seen block. If no blocks have been seen since btcd has +// connected, btcd is queried for the current block height and hash. +func GetCurBlock() (bs wallet.BlockStamp, err error) { + curBlock.RLock() + bs = curBlock.BlockStamp + curBlock.RUnlock() + if bs.Height != int32(btcutil.BlockHeightUnknown) { + return bs, nil + } + + // This is a hack and may result in races, but we need to make + // sure that btcd is connected and sending a message will succeed, + // or this will block forever. A better solution is to return an + // error to the reply handler immediately if btcd is disconnected. + if !btcdConnected.b { + return wallet.BlockStamp{ + Height: int32(btcutil.BlockHeightUnknown), + }, errors.New("current block unavailable") } n := <-NewJSONID - m, err := btcjson.CreateMessageWithId("getblockcount", - fmt.Sprintf("btcwallet(%v)", n)) - if err != nil { - // Can't continue. - return btcutil.BlockHeightUnknown + msg := btcjson.Message{ + Jsonrpc: "1.0", + Id: fmt.Sprintf("btcwallet(%v)", n), + Method: "getbestblock", } + m, _ := json.Marshal(msg) - c := make(chan int64) + c := make(chan *struct { + hash *btcwire.ShaHash + height int32 + }) replyHandlers.Lock() replyHandlers.m[n] = func(result interface{}, e *btcjson.Error) bool { if e != nil { - c <- btcutil.BlockHeightUnknown + c <- nil return true } - if balance, ok := result.(float64); ok { - c <- int64(balance) - } else { - c <- btcutil.BlockHeightUnknown + m, ok := result.(map[string]interface{}) + if !ok { + c <- nil + return true + } + hashBE, ok := m["hash"].(string) + if !ok { + c <- nil + return true + } + hash, err := btcwire.NewShaHashFromStr(hashBE) + if err != nil { + c <- nil + return true + } + fheight, ok := m["height"].(float64) + if !ok { + c <- nil + return true + } + c <- &struct { + hash *btcwire.ShaHash + height int32 + }{ + hash: hash, + height: int32(fheight), } return true } @@ -251,16 +297,22 @@ func getCurHeight() (height int64) { btcdMsgs <- m // Block until reply is ready. - height = <-c - curHeight.Lock() - if height > curHeight.h { - curHeight.h = height - } else { - height = curHeight.h + if reply := <-c; reply != nil { + curBlock.Lock() + if reply.height > curBlock.BlockStamp.Height { + bs = wallet.BlockStamp{ + Height: reply.height, + Hash: *reply.hash, + } + curBlock.BlockStamp = bs + } + curBlock.Unlock() + return bs, nil } - curHeight.Unlock() - return height + return wallet.BlockStamp{ + Height: int32(btcutil.BlockHeightUnknown), + }, errors.New("current block unavailable") } // CalculateBalance sums the amounts of all unspent transaction @@ -275,8 +327,8 @@ func getCurHeight() (height int64) { func (w *BtcWallet) CalculateBalance(confirms int) float64 { var bal uint64 // Measured in satoshi - height := getCurHeight() - if height == btcutil.BlockHeightUnknown { + bs, err := GetCurBlock() + if bs.Height == int32(btcutil.BlockHeightUnknown) || err != nil { return 0. } @@ -284,12 +336,12 @@ func (w *BtcWallet) CalculateBalance(confirms int) float64 { for _, u := range w.UtxoStore.s { // Utxos not yet in blocks (height -1) should only be // added if confirmations is 0. - if confirms == 0 || (u.Height != -1 && int(height-u.Height+1) >= confirms) { + if confirms == 0 || (u.Height != -1 && int(bs.Height-u.Height+1) >= confirms) { bal += u.Amt } } w.UtxoStore.RUnlock() - return float64(bal) / satoshiPerBTC + return float64(bal) / float64(btcutil.SatoshiPerBitcoin) } // Track requests btcd to send notifications of new transactions for @@ -305,7 +357,7 @@ func (w *BtcWallet) Track() { replyHandlers.m[n] = w.newBlockTxHandler replyHandlers.Unlock() for _, addr := range w.GetActiveAddresses() { - w.ReqNewTxsForAddress(addr) + w.ReqNewTxsForAddress(addr.Address) } n = <-NewJSONID @@ -323,21 +375,36 @@ func (w *BtcWallet) Track() { w.UtxoStore.RUnlock() } -// RescanForAddress requests btcd to rescan the blockchain for new -// transactions to addr. This is useful for making btcwallet catch up to -// a long-running btcd process, or for importing addresses and rescanning -// for unspent tx outputs. If len(blocks) is 0, the entire blockchain is -// rescanned. If len(blocks) is 1, the rescan will begin at height -// blocks[0]. If len(blocks) is 2 or greater, the rescan will be -// performed for the block range blocks[0]...blocks[1] (inclusive). -func (w *BtcWallet) RescanForAddress(addr string, blocks ...int) { - n := <-NewJSONID - params := []interface{}{addr} - if len(blocks) > 0 { - params = append(params, blocks[0]) +// RescanToBestBlock requests btcd to rescan the blockchain for new +// transactions to all wallet addresses. This is needed for making +// btcwallet catch up to a long-running btcd process, as otherwise +// it would have missed notifications as blocks are attached to the +// main chain. +func (w *BtcWallet) RescanToBestBlock() { + beginBlock := int32(0) + + if w.fullRescan { + // Need to perform a complete rescan since the wallet creation + // block. + beginBlock = w.CreatedAt() + log.Debugf("Rescanning account '%v' for new transactions since block height %v", + w.name, beginBlock) + } else { + // The last synced block height should be used the starting + // point for block rescanning. Grab the block stamp here. + bs := w.SyncedWith() + + log.Debugf("Rescanning account '%v' for new transactions since block height %v hash %v", + w.name, bs.Height, bs.Hash) + + // If we're synced with block x, must scan the blocks x+1 to best block. + beginBlock = bs.Height + 1 } - if len(blocks) > 1 { - params = append(params, blocks[1]) + + n := <-NewJSONID + params := []interface{}{ + beginBlock, + w.ActivePaymentAddresses(), } m := &btcjson.Message{ Jsonrpc: "1.0", @@ -349,18 +416,53 @@ func (w *BtcWallet) RescanForAddress(addr string, blocks ...int) { replyHandlers.Lock() replyHandlers.m[n] = func(result interface{}, e *btcjson.Error) bool { - // TODO(jrick) + // Rescan is compatible with new txs from connected block + // notifications, so use that handler. + _ = w.newBlockTxHandler(result, e) - // btcd returns a nil result when the rescan is complete. - // Returning true signals that this handler is finished - // and can be removed. - return result == nil + if result != nil { + // Notify frontends of new account balance. + confirmed := w.CalculateBalance(1) + unconfirmed := w.CalculateBalance(0) - confirmed + NotifyWalletBalance(frontendNotificationMaster, w.name, confirmed) + NotifyWalletBalanceUnconfirmed(frontendNotificationMaster, w.name, unconfirmed) + + return false + } + if bs, err := GetCurBlock(); err == nil { + w.SetSyncedWith(&bs) + w.dirty = true + if err = w.writeDirtyToDisk(); err != nil { + log.Errorf("cannot sync dirty wallet: %v", + err) + } + } + // If result is nil, the rescan has completed. Returning + // true removes this handler. + return true } replyHandlers.Unlock() btcdMsgs <- msg } +// ActivePaymentAddresses returns the second parameter for all rescan +// commands. The returned slice maps between payment address strings and +// the block height to begin rescanning for transactions to that address. +func (w *BtcWallet) ActivePaymentAddresses() []string { + w.mtx.RLock() + defer w.mtx.RUnlock() + + infos := w.GetActiveAddresses() + addrs := make([]string, len(infos)) + + for i := range infos { + addrs[i] = infos[i].Address + } + + return addrs +} + // ReqNewTxsForAddress sends a message to btcd to request tx updates // for addr for each new block that is added to the blockchain. func (w *BtcWallet) ReqNewTxsForAddress(addr string) { @@ -550,16 +652,17 @@ func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool // update the block height and hash. w.UtxoStore.RLock() for _, u := range w.UtxoStore.s { - if u.Height != -1 { - continue - } if bytes.Equal(u.Out.Hash[:], txhash[:]) && u.Out.Index == uint32(index) { - // Found it. + // Found a either a duplicate, or a change UTXO. If not change, + // ignore it. + if u.Height != -1 { + return false + } w.UtxoStore.RUnlock() w.UtxoStore.Lock() copy(u.BlockHash[:], blockhash[:]) - u.Height = int64(height) + u.Height = int32(height) w.UtxoStore.dirty = true w.UtxoStore.Unlock() @@ -571,9 +674,12 @@ func (w *BtcWallet) newBlockTxHandler(result interface{}, e *btcjson.Error) bool } w.UtxoStore.RUnlock() + // After iterating through all UTXOs, it was not a duplicate or + // change UTXO appearing in a block. Append a new Utxo to the end. + u := &tx.Utxo{ Amt: uint64(amt), - Height: int64(height), + Height: int32(height), Subscript: pkscript, } copy(u.Out.Hash[:], txhash[:]) @@ -638,14 +744,35 @@ func main() { // Open default wallet w, err := OpenWallet(cfg, "") - if err != nil { - log.Info(err.Error()) - } else { + switch err { + case ErrNoTxs: + // Do nothing special for now. This will be implemented when + // the tx history file is properly written. wallets.Lock() wallets.m[""] = w wallets.Unlock() + + case ErrNoUtxos: + // Add wallet, but mark wallet as needing a full rescan since + // the wallet creation block. This will take place when btcd + // connects. + wallets.Lock() + wallets.m[""] = w + wallets.Unlock() + w.fullRescan = true + + case nil: + wallets.Lock() + wallets.m[""] = w + wallets.Unlock() + + default: + log.Errorf("cannot open wallet: %v", err) } + // Start wallet disk syncer goroutine. + go DirtyWalletSyncer() + go func() { // Start HTTP server to listen and send messages to frontend and btcd // backend. Try reconnection if connection failed. diff --git a/cmdmgr.go b/cmdmgr.go index 6e9d4e4..cea05b4 100644 --- a/cmdmgr.go +++ b/cmdmgr.go @@ -19,6 +19,7 @@ package main import ( "encoding/hex" "encoding/json" + "errors" "fmt" "github.com/conformal/btcjson" "github.com/conformal/btcwallet/wallet" @@ -26,6 +27,13 @@ import ( "time" ) +var ( + // ErrBtcdDisconnected describes an error where an operation cannot + // successfully complete due to btcd not being connected to + // btcwallet. + ErrBtcdDisconnected = errors.New("btcd disconnected") +) + // ProcessFrontendMsg checks the message sent from a frontend. If the // message method is one that must be handled by btcwallet, the request // is processed here. Otherwise, the message is sent to btcd. @@ -129,9 +137,9 @@ func GetAddressesByAccount(reply chan []byte, msg *btcjson.Message) { return } - var result interface{} + var result []string if w := wallets.m[account]; w != nil { - result = w.Wallet.GetActiveAddresses() + result = w.ActivePaymentAddresses() } else { ReplyError(reply, msg.Id, &btcjson.ErrWalletInvalidAccountName) return @@ -689,7 +697,15 @@ func CreateEncryptedWallet(reply chan []byte, msg *btcjson.Message) { } else { net = btcwire.TestNet3 } - wlt, err := wallet.NewWallet(wname, desc, []byte(pass), net) + + bs, err := GetCurBlock() + if err != nil { + e := btcjson.ErrInternal + e.Message = "btcd disconnected" + ReplyError(reply, msg.Id, &e) + return + } + wlt, err := wallet.NewWallet(wname, desc, []byte(pass), net, &bs) if err != nil { log.Error("Error creating wallet: " + err.Error()) ReplyError(reply, msg.Id, &btcjson.ErrInternal) diff --git a/createtx.go b/createtx.go index 6cda5f1..6a3fd02 100644 --- a/createtx.go +++ b/createtx.go @@ -82,7 +82,10 @@ func (u ByAmount) Swap(i, j int) { // of all selected previous outputs. err will equal ErrInsufficientFunds if there // are not enough unspent outputs to spend amt. func selectInputs(s tx.UtxoStore, amt uint64, minconf int) (inputs []*tx.Utxo, btcout uint64, err error) { - height := getCurHeight() + bs, err := GetCurBlock() + if err != nil { + return nil, 0, err + } // Create list of eligible unspent previous outputs to use as tx // inputs, and sort by the amount in reverse order so a minimum number @@ -93,7 +96,7 @@ func selectInputs(s tx.UtxoStore, amt uint64, minconf int) (inputs []*tx.Utxo, b // to a change address, resulting in a UTXO not yet mined in a block. // For now, disallow creating transactions until these UTXOs are mined // into a block and show up as part of the balance. - if utxo.Height != -1 && int(height-utxo.Height) >= minconf { + if utxo.Height != -1 && int(bs.Height-utxo.Height) >= minconf { eligible = append(eligible, utxo) } } diff --git a/createtx_test.go b/createtx_test.go index f9cce7b..a04eb98 100644 --- a/createtx_test.go +++ b/createtx_test.go @@ -1,10 +1,6 @@ package main import ( - "encoding/hex" - "encoding/json" - "fmt" - "github.com/conformal/btcjson" "github.com/conformal/btcscript" "github.com/conformal/btcutil" "github.com/conformal/btcwallet/tx" @@ -15,7 +11,8 @@ import ( func TestFakeTxs(t *testing.T) { // First we need a wallet. - w, err := wallet.NewWallet("banana wallet", "", []byte("banana"), btcwire.MainNet) + w, err := wallet.NewWallet("banana wallet", "", []byte("banana"), + btcwire.MainNet, &wallet.BlockStamp{}) if err != nil { t.Errorf("Can not create encrypted wallet: %s", err) return @@ -58,30 +55,15 @@ func TestFakeTxs(t *testing.T) { btcw.UtxoStore.s = append(btcw.UtxoStore.s, utxo) // Fake our current block height so btcd doesn't need to be queried. - curHeight.h = 12346 + curBlock.BlockStamp.Height = 12346 // Create the transaction. pairs := map[string]uint64{ "17XhEvq9Nahdj7Xe1nv6oRe1tEmaHUuynH": 5000, } - createdTx, err := btcw.txToPairs(pairs, 100, 0) + _, err = btcw.txToPairs(pairs, 100, 0) if err != nil { t.Errorf("Tx creation failed: %s", err) return } - - msg := btcjson.Message{ - Jsonrpc: "1.0", - Id: "test", - Method: "sendrawtransaction", - Params: []interface{}{ - hex.EncodeToString(createdTx.rawTx), - }, - } - m, _ := json.Marshal(msg) - _ = m - _ = fmt.Println - - // Uncomment to print out json to send raw transaction - // fmt.Println(string(m)) } diff --git a/disksync.go b/disksync.go index 997d815..6f5639c 100644 --- a/disksync.go +++ b/disksync.go @@ -20,9 +20,47 @@ import ( "fmt" "os" "path/filepath" + "sync" "time" ) +var ( + // dirtyWallets holds a set of wallets that include dirty components. + dirtyWallets = struct { + sync.Mutex + m map[*BtcWallet]bool + }{ + m: make(map[*BtcWallet]bool), + } +) + +// DirtyWalletSyncer synces dirty wallets for cases where the updated +// information was not required to be immediately written to disk. Wallets +// may be added to dirtyWallets and will be checked and processed every 10 +// seconds by this function. +// +// This never returns and is meant to be called from a goroutine. +func DirtyWalletSyncer() { + ticker := time.Tick(10 * time.Second) + for { + select { + case <-ticker: + dirtyWallets.Lock() + for w := range dirtyWallets.m { + log.Debugf("Syncing wallet '%v' to disk", + w.Wallet.Name()) + if err := w.writeDirtyToDisk(); err != nil { + log.Errorf("cannot sync dirty wallet: %v", + err) + } else { + delete(dirtyWallets.m, w) + } + } + dirtyWallets.Unlock() + } + } +} + // writeDirtyToDisk checks for the dirty flag on an account's wallet, // txstore, and utxostore, writing them to disk if any are dirty. func (w *BtcWallet) writeDirtyToDisk() error { diff --git a/sockets.go b/sockets.go index db0a17e..74bb4fc 100644 --- a/sockets.go +++ b/sockets.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "github.com/conformal/btcjson" + "github.com/conformal/btcwallet/wallet" "github.com/conformal/btcwire" "net" "net/http" @@ -110,8 +111,8 @@ func frontendListenerDuplicator() { // place these notifications in that function. NotifyBtcdConnected(frontendNotificationMaster, btcdConnected.b) - if btcdConnected.b { - NotifyNewBlockChainHeight(c, getCurHeight()) + if bs, err := GetCurBlock(); err == nil { + NotifyNewBlockChainHeight(c, bs.Height) NotifyBalances(c) } @@ -144,6 +145,7 @@ func frontendListenerDuplicator() { } } +// NotifyBtcdConnected notifies all frontends of a new btcd connection. func NotifyBtcdConnected(reply chan []byte, conn bool) { btcdConnected.b = conn var idStr interface{} = "btcwallet:btcdconnected" @@ -266,11 +268,17 @@ func ProcessBtcdNotificationReply(b []byte) { log.Errorf("Unable to unmarshal btcd message: %v", err) return } + if r.Id == nil { + // btcd should only ever be sending JSON messages with a string in + // the id field. Log the error and drop the message. + log.Error("Unable to process btcd notification or reply.") + return + } idStr, ok := (*r.Id).(string) if !ok { // btcd should only ever be sending JSON messages with a string in // the id field. Log the error and drop the message. - log.Error("Unable to process btcd notification or reply.") + log.Error("Incorrect btcd notification id type.") return } @@ -338,7 +346,7 @@ func ProcessBtcdNotificationReply(b []byte) { // NotifyNewBlockChainHeight notifies all frontends of a new // blockchain height. -func NotifyNewBlockChainHeight(reply chan []byte, height int64) { +func NotifyNewBlockChainHeight(reply chan []byte, height int32) { var id interface{} = "btcwallet:newblockchainheight" msgRaw := &btcjson.Reply{ Result: height, @@ -372,7 +380,7 @@ func NtfnBlockConnected(r interface{}) { log.Error("blockconnected notification: invalid height") return } - height := int64(heightf) + height := int32(heightf) var minedTxs []string if iminedTxs, ok := result["minedtxs"].([]interface{}); ok { minedTxs = make([]string, len(iminedTxs)) @@ -386,12 +394,35 @@ func NtfnBlockConnected(r interface{}) { } } - curHeight.Lock() - curHeight.h = height - curHeight.Unlock() + curBlock.Lock() + curBlock.BlockStamp = wallet.BlockStamp{ + Height: height, + Hash: *hash, + } + curBlock.Unlock() - // TODO(jrick): update TxStore and UtxoStore with new hash - _ = hash + // btcd notifies btcwallet about transactions first, and then sends + // the block notification. This prevents any races from saving a + // synced-to block before all notifications from the block have been + // processed. + bs := &wallet.BlockStamp{ + Height: height, + Hash: *hash, + } + for _, w := range wallets.m { + // We do not write synced info immediatelly out to disk. + // If btcd is performing an IBD, that would result in + // writing out the wallet to disk for each processed block. + // Instead, mark as dirty and let another goroutine process + // the dirty wallet. + w.mtx.Lock() + w.Wallet.SetSyncedWith(bs) + w.dirty = true + w.mtx.Unlock() + dirtyWallets.Lock() + dirtyWallets.m[w] = true + dirtyWallets.Unlock() + } // Notify frontends of new blockchain height. NotifyNewBlockChainHeight(frontendNotificationMaster, height) @@ -451,7 +482,7 @@ func NtfnBlockDisconnected(r interface{}) { if !ok { log.Error("blockdisconnected notification: invalid height") } - height := int64(heightf) + height := int32(heightf) // Rollback Utxo and Tx data stores. go func() { @@ -571,21 +602,25 @@ func BtcdHandshake(ws *websocket.Conn) { return } + // TODO(jrick): Check that there was not any reorgs done + // since last connection. If so, rollback and rescan to + // catch up. + + for _, w := range wallets.m { + w.RescanToBestBlock() + } + // Begin tracking wallets against this btcd instance. for _, w := range wallets.m { w.Track() } - // Request the new block height, and notify frontends. - // - // TODO(jrick): Check that there was not any reorgs done - // since last connection. - NotifyNewBlockChainHeight(frontendNotificationMaster, getCurHeight()) - - // Notify frontends of all account balances, calculated based - // from the block height of this new btcd connection. - NotifyBalances(frontendNotificationMaster) - // (Re)send any unmined transactions to btcd in case of a btcd restart. resendUnminedTxs() + + // Get current blockchain height and best block hash. + if bs, err := GetCurBlock(); err == nil { + NotifyNewBlockChainHeight(frontendNotificationMaster, bs.Height) + NotifyBalances(frontendNotificationMaster) + } } diff --git a/tx/tx.go b/tx/tx.go index 431f823..b12b8bc 100644 --- a/tx/tx.go +++ b/tx/tx.go @@ -45,7 +45,7 @@ type Utxo struct { Amt uint64 // Measured in Satoshis // Height is -1 if Utxo has not yet appeared in a block. - Height int64 + Height int32 // BlockHash is zeroed if Utxo has not yet appeared in a block. BlockHash btcwire.ShaHash @@ -66,7 +66,7 @@ type TxStore []interface{} type RecvTx struct { TxHash btcwire.ShaHash BlockHash btcwire.ShaHash - Height int64 + Height int32 Amt uint64 // Measured in Satoshis SenderAddr [ripemd160.Size]byte ReceiverAddr [ripemd160.Size]byte @@ -77,7 +77,7 @@ type RecvTx struct { type SendTx struct { TxHash btcwire.ShaHash BlockHash btcwire.ShaHash - Height int64 + Height int32 Fee uint64 // Measured in Satoshis SenderAddr [ripemd160.Size]byte ReceiverAddrs []struct { @@ -156,7 +156,7 @@ func (u *UtxoStore) WriteTo(w io.Writer) (n int64, err error) { // // Correct results rely on u being sorted by block height in // increasing order. -func (u *UtxoStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool) { +func (u *UtxoStore) Rollback(height int32, hash *btcwire.ShaHash) (modified bool) { s := *u // endlen specifies the final length of the rolled-back UtxoStore. @@ -219,7 +219,7 @@ func (u *UtxoStore) Remove(toRemove []*Utxo) (modified bool) { // ReadFrom satisifies the io.ReaderFrom interface. A Utxo is read // from r with the format: // -// [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (8 bytes), BlockHash (32 bytes)] +// [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (4 bytes), BlockHash (32 bytes)] // // Each field is read little endian. func (u *Utxo) ReadFrom(r io.Reader) (n int64, err error) { @@ -249,7 +249,7 @@ func (u *Utxo) ReadFrom(r io.Reader) (n int64, err error) { // WriteTo satisifies the io.WriterTo interface. A Utxo is written to // w in the format: // -// [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (8 bytes), BlockHash (32 bytes)] +// [AddrHash (20 bytes), Out (36 bytes), Subscript (varies), Amt (8 bytes), Height (4 bytes), BlockHash (32 bytes)] // // Each field is written little endian. func (u *Utxo) WriteTo(w io.Writer) (n int64, err error) { @@ -452,7 +452,7 @@ func (txs *TxStore) WriteTo(w io.Writer) (n int64, err error) { // // Correct results rely on txs being sorted by block height in // increasing order. -func (txs *TxStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool) { +func (txs *TxStore) Rollback(height int32, hash *btcwire.ShaHash) (modified bool) { s := ([]interface{})(*txs) // endlen specifies the final length of the rolled-back TxStore. @@ -470,7 +470,7 @@ func (txs *TxStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool }() for i := len(s) - 1; i >= 0; i-- { - var txheight int64 + var txheight int32 var txhash *btcwire.ShaHash switch s[i].(type) { case *RecvTx: @@ -498,7 +498,7 @@ func (txs *TxStore) Rollback(height int64, hash *btcwire.ShaHash) (modified bool // ReadFrom satisifies the io.ReaderFrom interface. A RecTx is read // in from r with the format: // -// [TxHash (32 bytes), BlockHash (32 bytes), Height (8 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)] +// [TxHash (32 bytes), BlockHash (32 bytes), Height (4 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)] // // Each field is read little endian. func (tx *RecvTx) ReadFrom(r io.Reader) (n int64, err error) { @@ -524,7 +524,7 @@ func (tx *RecvTx) ReadFrom(r io.Reader) (n int64, err error) { // WriteTo satisifies the io.WriterTo interface. A RecvTx is written to // w in the format: // -// [TxHash (32 bytes), BlockHash (32 bytes), Height (8 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)] +// [TxHash (32 bytes), BlockHash (32 bytes), Height (4 bytes), Amt (8 bytes), SenderAddr (20 bytes), ReceiverAddr (20 bytes)] // // Each field is written little endian. func (tx *RecvTx) WriteTo(w io.Writer) (n int64, err error) { @@ -550,7 +550,7 @@ func (tx *RecvTx) WriteTo(w io.Writer) (n int64, err error) { // ReadFrom satisifies the io.WriterTo interface. A SendTx is read // from r with the format: // -// [TxHash (32 bytes), Height (8 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...] +// [TxHash (32 bytes), Height (4 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...] // // Each field is read little endian. func (tx *SendTx) ReadFrom(r io.Reader) (n int64, err error) { @@ -599,7 +599,7 @@ func (tx *SendTx) ReadFrom(r io.Reader) (n int64, err error) { // WriteTo satisifies the io.WriterTo interface. A SendTx is written to // w in the format: // -// [TxHash (32 bytes), Height (8 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...] +// [TxHash (32 bytes), Height (4 bytes), Fee (8 bytes), SenderAddr (20 bytes), len(ReceiverAddrs) (4 bytes), ReceiverAddrs[Addr (20 bytes), Amt (8 bytes)]...] // // Each field is written little endian. func (tx *SendTx) WriteTo(w io.Writer) (n int64, err error) { diff --git a/tx/tx_test.go b/tx/tx_test.go index 99565ef..fdc5b39 100644 --- a/tx/tx_test.go +++ b/tx/tx_test.go @@ -144,7 +144,7 @@ func TestUtxoStoreWriteRead(t *testing.T) { utxo.Out.Index = uint32(i + 2) utxo.Subscript = []byte{} utxo.Amt = uint64(i + 3) - utxo.Height = int64(i + 4) + utxo.Height = int32(i + 4) *store1 = append(*store1, utxo) } diff --git a/wallet/wallet.go b/wallet/wallet.go index ff0dbfd..02c364a 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -34,7 +34,6 @@ import ( "github.com/davecgh/go-spew/spew" "hash" "io" - "math" "math/big" "sync" "time" @@ -326,16 +325,22 @@ func (v *varEntries) ReadFrom(r io.Reader) (n int64, err error) { // from and write to any type of byte streams, including files. // TODO(jrick) remove as many more magic numbers as possible. type Wallet struct { - version uint32 - net btcwire.BitcoinNet - flags walletFlags - uniqID [6]byte - createDate int64 - name [32]byte - desc [256]byte - highestUsed int64 - kdfParams kdfParameters - keyGenerator btcAddress + version uint32 + net btcwire.BitcoinNet + flags walletFlags + uniqID [6]byte + createDate int64 + name [32]byte + desc [256]byte + highestUsed int64 + kdfParams kdfParameters + keyGenerator btcAddress + + // These are non-standard and fit in the extra 1024 bytes between the + // root address and the appended entries. + syncedBlockHeight int32 + syncedBlockHash btcwire.ShaHash + addrMap map[[ripemd160.Size]byte]*btcAddress addrCommentMap map[[ripemd160.Size]byte]*[]byte txCommentMap map[[sha256.Size]byte]*[]byte @@ -349,11 +354,22 @@ type Wallet struct { lastChainIdx int64 } +// UnusedWalletBytes specifies the number of actually unused bytes +// between the root address and the appended entries in a serialized +// wallet. Armory's wallet file format provides 1024 unused bytes +// in this space. btcwallet requires saving a few additional details +// with the wallet file, so the binary sizes of those are subtracted +// from 1024. Currently, these are: +// +// - last synced block height (int32, 4 bytes) +// - last synced block hash (btcwire.ShaHash, btcwire.HashSize bytes) +const UnusedWalletBytes = 1024 - 4 - btcwire.HashSize + // NewWallet creates and initializes a new Wallet. name's and // desc's binary representation must not exceed 32 and 256 bytes, // respectively. All address private keys are encrypted with passphrase. // The wallet is returned unlocked. -func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*Wallet, error) { +func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet, createdAt *BlockStamp) (*Wallet, error) { if binary.Size(name) > 32 { return nil, errors.New("name exceeds 32 byte maximum size") } @@ -366,7 +382,7 @@ func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*W rootkey, chaincode := make([]byte, 32), make([]byte, 32) rand.Read(rootkey) rand.Read(chaincode) - root, err := newRootBtcAddress(rootkey, nil, chaincode) + root, err := newRootBtcAddress(rootkey, nil, chaincode, createdAt) if err != nil { return nil, err } @@ -387,15 +403,17 @@ func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*W useEncryption: true, watchingOnly: false, }, - createDate: time.Now().Unix(), - highestUsed: -1, - kdfParams: *kdfp, - keyGenerator: *root, - addrMap: make(map[[ripemd160.Size]byte]*btcAddress), - addrCommentMap: make(map[[ripemd160.Size]byte]*[]byte), - txCommentMap: make(map[[sha256.Size]byte]*[]byte), - chainIdxMap: make(map[int64]*[ripemd160.Size]byte), - lastChainIdx: pregenerated - 1, + createDate: time.Now().Unix(), + highestUsed: -1, + kdfParams: *kdfp, + keyGenerator: *root, + syncedBlockHeight: createdAt.Height, + syncedBlockHash: createdAt.Hash, + addrMap: make(map[[ripemd160.Size]byte]*btcAddress), + addrCommentMap: make(map[[ripemd160.Size]byte]*[]byte), + txCommentMap: make(map[[sha256.Size]byte]*[]byte), + chainIdxMap: make(map[int64]*[ripemd160.Size]byte), + lastChainIdx: pregenerated - 1, } // Add root address to maps. @@ -410,7 +428,7 @@ func NewWallet(name, desc string, passphrase []byte, net btcwire.BitcoinNet) (*W if err != nil { return nil, err } - newaddr, err := newBtcAddress(privkey, nil) + newaddr, err := newBtcAddress(privkey, nil, createdAt) if err != nil { return nil, err } @@ -464,7 +482,9 @@ func (w *Wallet) ReadFrom(r io.Reader) (n int64, err error) { &w.kdfParams, make([]byte, 256), &w.keyGenerator, - make([]byte, 1024), + &w.syncedBlockHeight, + &w.syncedBlockHash, + make([]byte, UnusedWalletBytes), &appendedEntries, } for _, data := range datas { @@ -558,7 +578,9 @@ func (w *Wallet) WriteTo(wtr io.Writer) (n int64, err error) { &w.kdfParams, make([]byte, 256), &w.keyGenerator, - make([]byte, 1024), + &w.syncedBlockHeight, + &w.syncedBlockHash, + make([]byte, UnusedWalletBytes), &appendedEntries, } var written int64 @@ -630,9 +652,10 @@ func (w *Wallet) Version() (string, int) { return "", 0 } -// NextUnusedAddress attempts to get the next chained address. It -// currently relies on pre-generated addresses and will return an empty -// string if the address pool has run out. TODO(jrick) +// NextUnusedAddress attempts to get the next chained address. +// +// TODO(jrick): this currently relies on pre-generated addresses +// and will return an empty string if the address pool has run out. func (w *Wallet) NextUnusedAddress() (string, error) { _ = w.lastChainIdx w.highestUsed++ @@ -701,6 +724,29 @@ func (w *Wallet) Net() btcwire.BitcoinNet { return w.net } +// SetSyncedWith marks the wallet to be in sync with the block +// described by height and hash. +func (w *Wallet) SetSyncedWith(bs *BlockStamp) { + w.syncedBlockHeight = bs.Height + copy(w.syncedBlockHash[:], bs.Hash[:]) +} + +// SyncedWith returns the height and hash of the block the wallet is +// currently marked to be in sync with. +func (w *Wallet) SyncedWith() *BlockStamp { + return &BlockStamp{ + Height: w.syncedBlockHeight, + Hash: w.syncedBlockHash, + } +} + +// CreatedAt returns the height of the blockchain at the time of wallet +// creation. This is needed when performaing a full rescan to prevent +// unnecessary rescanning before wallet addresses first appeared. +func (w *Wallet) CreatedAt() int32 { + return w.keyGenerator.firstBlock +} + func (w *Wallet) addr160ForIdx(idx int64) (*[ripemd160.Size]byte, error) { if idx > w.lastChainIdx { return nil, errors.New("chain index out of range") @@ -708,21 +754,27 @@ func (w *Wallet) addr160ForIdx(idx int64) (*[ripemd160.Size]byte, error) { return w.chainIdxMap[idx], nil } +// AddressInfo holds information regarding an address needed to manage +// a complete wallet. +type AddressInfo struct { + Address string + FirstBlock int32 +} + // GetActiveAddresses returns all wallet addresses that have been // requested to be generated. These do not include pre-generated // addresses. -func (w *Wallet) GetActiveAddresses() []string { - addrs := []string{} +func (w *Wallet) GetActiveAddresses() []*AddressInfo { + addrs := make([]*AddressInfo, 0, w.highestUsed+1) for i := int64(-1); i <= w.highestUsed; i++ { addr160, err := w.addr160ForIdx(i) if err != nil { return addrs } addr := w.addrMap[*addr160] - addrstr, err := addr.paymentAddress(w.net) - // TODO(jrick): propigate error + info, err := addr.info(w.Net()) if err == nil { - addrs = append(addrs, addrstr) + addrs = append(addrs, info) } } return addrs @@ -817,7 +869,7 @@ type btcAddress struct { // newBtcAddress initializes and returns a new address. privkey must // be 32 bytes. iv must be 16 bytes, or nil (in which case it is // randomly generated). -func newBtcAddress(privkey, iv []byte) (addr *btcAddress, err error) { +func newBtcAddress(privkey, iv []byte, bs *BlockStamp) (addr *btcAddress, err error) { if len(privkey) != 32 { return nil, errors.New("private key is not 32 bytes") } @@ -834,8 +886,8 @@ func newBtcAddress(privkey, iv []byte) (addr *btcAddress, err error) { hasPrivKey: true, hasPubKey: true, }, - firstSeen: math.MaxInt64, - firstBlock: math.MaxInt32, + firstSeen: time.Now().Unix(), + firstBlock: bs.Height, } copy(addr.initVector[:], iv) pub := pubkeyFromPrivkey(privkey) @@ -848,12 +900,12 @@ func newBtcAddress(privkey, iv []byte) (addr *btcAddress, err error) { // newRootBtcAddress generates a new address, also setting the // chaincode and chain index to represent this address as a root // address. -func newRootBtcAddress(privKey, iv, chaincode []byte) (addr *btcAddress, err error) { +func newRootBtcAddress(privKey, iv, chaincode []byte, bs *BlockStamp) (addr *btcAddress, err error) { if len(chaincode) != 32 { return nil, errors.New("chaincode is not 32 bytes") } - addr, err = newBtcAddress(privKey, iv) + addr, err = newBtcAddress(privKey, iv, bs) if err != nil { return nil, err } @@ -1039,6 +1091,20 @@ func (a *btcAddress) paymentAddress(net btcwire.BitcoinNet) (string, error) { return btcutil.EncodeAddress(a.pubKeyHash[:], net) } +// info returns information about a btcAddress stored in a AddressInfo +// struct. +func (a *btcAddress) info(net btcwire.BitcoinNet) (*AddressInfo, error) { + address, err := a.paymentAddress(net) + if err != nil { + return nil, err + } + + return &AddressInfo{ + Address: address, + FirstBlock: a.firstBlock, + }, nil +} + func walletHash(b []byte) uint32 { sum := btcwire.DoubleSha256(b) return binary.LittleEndian.Uint32(sum) @@ -1322,3 +1388,11 @@ func (e *deletedEntry) ReadFrom(r io.Reader) (n int64, err error) { } return n + int64(nRead), err } + +// BlockStamp defines a block (by height and a unique hash) and is +// used to mark a point in the blockchain that a wallet element is +// synced to. +type BlockStamp struct { + Height int32 + Hash btcwire.ShaHash +} diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 27dc5e7..fbaeab8 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -79,7 +79,9 @@ func TestBtcAddressSerializer(t *testing.T) { } func TestWalletCreationSerialization(t *testing.T) { - w1, err := NewWallet("banana wallet", "A wallet for testing.", []byte("banana"), btcwire.MainNet) + createdAt := &BlockStamp{} + w1, err := NewWallet("banana wallet", "A wallet for testing.", + []byte("banana"), btcwire.MainNet, createdAt) if err != nil { t.Error("Error creating new wallet: " + err.Error()) }