Add wtxmgr package.

wtxmgr/README.md

@@ -0,0 +1,45 @@
+wtxmgr
+======
+
+[![Build Status](https://travis-ci.org/btcsuite/btcwallet.png?branch=master)]
+(https://travis-ci.org/btcsuite/btcwallet)
+
+Package wtxmgr provides storage and spend tracking of wallet transactions and
+their relevant input and outputs.
+
+## Feature overview
+
+- Storage for relevant wallet transactions
+- Ability to mark outputs as controlled by wallet
+- Unspent transaction output index
+- Balance tracking
+- Automatic spend tracking for transaction inserts and removals
+- Double spend detection and correction after blockchain reorgs
+- Scalable design:
+  - Utilizes similar prefixes to allow cursor iteration over relevant transaction
+    inputs and outputs
+  - Programmatically detectable errors, including encapsulation of errors from
+    packages it relies on
+  - Operates under its own walletdb namespace
+    
+## Documentation
+
+[![GoDoc](https://godoc.org/github.com/btcsuite/btcwallet/wtxmgr?status.png)]
+(http://godoc.org/github.com/btcsuite/btcwallet/wtxmgr)
+
+Full `go doc` style documentation for the project can be viewed online without
+installing this package by using the GoDoc site here:
+http://godoc.org/github.com/btcsuite/btcwallet/wtxmgr
+
+You can also view the documentation locally once the package is installed with
+the `godoc` tool by running `godoc -http=":6060"` and pointing your browser to
+http://localhost:6060/pkg/github.com/btcsuite/btcwallet/wtxmgr
+
+## Installation
+
+```bash
+$ go get github.com/btcsuite/btcwallet/wtxmgr
+```
+
+Package wtxmgr is licensed under the [copyfree](http://copyfree.org) ISC
+License.

wtxmgr/doc.go

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2013-2015 Conformal Systems LLC <info@conformal.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+// Package wtxmgr provides an implementation of a transaction database handling
+// spend tracking for a bitcoin wallet.  Its primary purpose is to save
+// transactions with outputs spendable with wallet keys and transactions that
+// are signed by wallet keys in memory, handle spend tracking for unspent
+// outputs and newly-inserted transactions, and report the spendable balance
+// from each unspent transaction output. It uses walletdb as the backend for
+// storing the serialized transaction objects in buckets.
+//
+// Transaction outputs which are spendable by wallet keys are called credits
+// (because they credit to a wallet's total spendable balance).  Transaction
+// inputs which spend previously-inserted credits are called debits (because
+// they debit from the wallet's spendable balance).
+//
+// Spend tracking is mostly automatic.  When a new transaction is inserted, if
+// it spends from any unspent credits, they are automatically marked spent by
+// the new transaction, and each input which spent a credit is marked as a
+// debit.  However, transaction outputs of inserted transactions must manually
+// marked as credits, as this package has no knowledge of wallet keys or
+// addresses, and therefore cannot determine which outputs may be spent.
+//
+// Details regarding individual transactions and their credits and debits may be
+// queried either by just a transaction hash, or by hash and block.  When
+// querying for just a transaction hash, the most recent transaction with a
+// matching hash will be queried.  However, because transaction hashes may
+// collide with other transaction hashes, methods to query for specific
+// transactions in the chain (or unmined) are provided as well.
+package wtxmgr

wtxmgr/error.go

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2015 Conformal Systems LLC <info@conformal.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package wtxmgr
+
+import "fmt"
+
+// ErrorCode identifies a category of error.
+type ErrorCode uint8
+
+// These constants are used to identify a specific Error.
+const (
+	// ErrDatabase indicates an error with the underlying database.  When
+	// this error code is set, the Err field of the Error will be
+	// set to the underlying error returned from the database.
+	ErrDatabase ErrorCode = iota
+
+	// ErrData describes an error where data stored in the transaction
+	// database is incorrect.  This may be due to missing values, values of
+	// wrong sizes, or data from different buckets that is inconsistent with
+	// itself.  Recovering from an ErrData requires rebuilding all
+	// transaction history or manual database surgery.  If the failure was
+	// not due to data corruption, this error category indicates a
+	// programming error in this package.
+	ErrData
+
+	// ErrInput describes an error where the variables passed into this
+	// function by the caller are obviously incorrect.  Examples include
+	// passing transactions which do not serialize, or attempting to insert
+	// a credit at an index for which no transaction output exists.
+	ErrInput
+
+	// ErrAlreadyExists describes an error where creating the store cannot
+	// continue because a store already exists in the namespace.
+	ErrAlreadyExists
+
+	// ErrNoExists describes an error where the store cannot be opened due to
+	// it not already existing in the namespace.  This error should be
+	// handled by creating a new store.
+	ErrNoExists
+
+	// ErrUnknownVersion describes an error where the store already exists
+	// but the database version is newer than latest version known to this
+	// software.  This likely indicates an outdated binary.
+	ErrUnknownVersion
+)
+
+var errStrs = [...]string{
+	ErrDatabase:       "ErrDatabase",
+	ErrData:           "ErrData",
+	ErrInput:          "ErrInput",
+	ErrAlreadyExists:  "ErrAlreadyExists",
+	ErrNoExists:       "ErrNoExists",
+	ErrUnknownVersion: "ErrUnknownVersion",
+}
+
+// String returns the ErrorCode as a human-readable name.
+func (e ErrorCode) String() string {
+	if e < ErrorCode(len(errStrs)) {
+		return errStrs[e]
+	}
+	return fmt.Sprintf("ErrorCode(%d)", e)
+}
+
+// Error provides a single type for errors that can happen during Store
+// operation.
+type Error struct {
+	Code ErrorCode // Describes the kind of error
+	Desc string    // Human readable description of the issue
+	Err  error     // Underlying error, optional
+}
+
+// Error satisfies the error interface and prints human-readable errors.
+func (e Error) Error() string {
+	if e.Err != nil {
+		return e.Desc + ": " + e.Err.Error()
+	}
+	return e.Desc
+}
+
+func storeError(c ErrorCode, desc string, err error) Error {
+	return Error{Code: c, Desc: desc, Err: err}
+}
+
+// IsNoExists returns whether an error is a Error with the ErrNoExists error
+// code.
+func IsNoExists(err error) bool {
+	serr, ok := err.(Error)
+	return ok && serr.Code == ErrNoExists
+}

wtxmgr/example_test.go

@@ -0,0 +1,233 @@
+// Copyright (c) 2015 Conformal Systems LLC <info@conformal.com>
+//
+// Permission to use, copy, modify, and distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package wtxmgr_test
+
+import (
+	"fmt"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcwallet/wtxmgr"
+)
+
+var (
+	// Spends: bogus
+	// Outputs: 10 BTC
+	exampleTxRecordA *wtxmgr.TxRecord
+
+	// Spends: A:0
+	// Outputs: 5 BTC, 5 BTC
+	exampleTxRecordB *wtxmgr.TxRecord
+)
+
+func init() {
+	tx := spendOutput(&wire.ShaHash{}, 0, 10e8)
+	rec, err := wtxmgr.NewTxRecordFromMsgTx(tx, timeNow())
+	if err != nil {
+		panic(err)
+	}
+	exampleTxRecordA = rec
+
+	tx = spendOutput(&exampleTxRecordA.Hash, 0, 5e8, 5e8)
+	rec, err = wtxmgr.NewTxRecordFromMsgTx(tx, timeNow())
+	if err != nil {
+		panic(err)
+	}
+	exampleTxRecordB = rec
+}
+
+var exampleBlock100 = makeBlockMeta(100)
+
+// This example demonstrates reporting the Store balance given an unmined and
+// mined transaction given 0, 1, and 6 block confirmations.
+func ExampleStore_Balance() {
+	s, teardown, err := testStore()
+	defer teardown()
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Prints balances for 0 block confirmations, 1 confirmation, and 6
+	// confirmations.
+	printBalances := func(syncHeight int32) {
+		zeroConfBal, err := s.Balance(0, syncHeight)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		oneConfBal, err := s.Balance(1, syncHeight)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		sixConfBal, err := s.Balance(6, syncHeight)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		fmt.Printf("%v, %v, %v\n", zeroConfBal, oneConfBal, sixConfBal)
+	}
+
+	// Insert a transaction which outputs 10 BTC unmined and mark the output
+	// as a credit.
+	err = s.InsertTx(exampleTxRecordA, nil)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	err = s.AddCredit(exampleTxRecordA, nil, 0, false)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	printBalances(100)
+
+	// Mine the transaction in block 100 and print balances again with a
+	// sync height of 100 and 105 blocks.
+	err = s.InsertTx(exampleTxRecordA, &exampleBlock100)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	printBalances(100)
+	printBalances(105)
+
+	// Output:
+	// 10 BTC, 0 BTC, 0 BTC
+	// 10 BTC, 10 BTC, 0 BTC
+	// 10 BTC, 10 BTC, 10 BTC
+}
+
+func ExampleStore_Rollback() {
+	s, teardown, err := testStore()
+	defer teardown()
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Insert a transaction which outputs 10 BTC in a block at height 100.
+	err = s.InsertTx(exampleTxRecordA, &exampleBlock100)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Rollback everything from block 100 onwards.
+	err = s.Rollback(100)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Assert that the transaction is now unmined.
+	details, err := s.TxDetails(&exampleTxRecordA.Hash)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if details == nil {
+		fmt.Println("No details found")
+		return
+	}
+	fmt.Println(details.Block.Height)
+
+	// Output:
+	// -1
+}
+
+func Example_basicUsage() {
+	// Open the database.
+	db, dbTeardown, err := testDB()
+	defer dbTeardown()
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Create or open a db namespace for the transaction store.
+	ns, err := db.Namespace([]byte("txstore"))
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Create (or open) the transaction store in the provided namespace.
+	s, err := wtxmgr.Create(ns)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Insert an unmined transaction that outputs 10 BTC to a wallet address
+	// at output 0.
+	err = s.InsertTx(exampleTxRecordA, nil)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	err = s.AddCredit(exampleTxRecordA, nil, 0, false)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Insert a second transaction which spends the output, and creates two
+	// outputs.  Mark the second one (5 BTC) as wallet change.
+	err = s.InsertTx(exampleTxRecordB, nil)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	err = s.AddCredit(exampleTxRecordB, nil, 1, true)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Mine each transaction in a block at height 100.
+	err = s.InsertTx(exampleTxRecordA, &exampleBlock100)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	err = s.InsertTx(exampleTxRecordB, &exampleBlock100)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// Print the one confirmation balance.
+	bal, err := s.Balance(1, 100)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(bal)
+
+	// Fetch unspent outputs.
+	utxos, err := s.UnspentOutputs()
+	if err != nil {
+		fmt.Println(err)
+	}
+	expectedOutPoint := wire.OutPoint{Hash: exampleTxRecordB.Hash, Index: 1}
+	for _, utxo := range utxos {
+		fmt.Println(utxo.OutPoint == expectedOutPoint)
+	}
+
+	// Output:
+	// 5 BTC
+	// true
+}

wtxmgr/log.go

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2013-2015 Conformal Systems LLC <info@conformal.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package wtxmgr
+
+import "github.com/btcsuite/btclog"
+
+// log is a logger that is initialized with no output filters.  This
+// means the package will not perform any logging by default until the caller
+// requests it.
+var log = btclog.Disabled
+
+// DisableLog disables all library log output.  Logging output is disabled
+// by default until either UseLogger or SetLogWriter are called.
+func DisableLog() {
+	log = btclog.Disabled
+}
+
+// UseLogger uses a specified Logger to output package logging info.
+// This should be used in preference to SetLogWriter if the caller is also
+// using btclog.
+func UseLogger(logger btclog.Logger) {
+	log = logger
+}

wtxmgr/query.go

@@ -0,0 +1,474 @@
+/*
+ * Copyright (c) 2015 Conformal Systems LLC <info@conformal.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package wtxmgr
+
+import (
+	"fmt"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/btcsuite/btcwallet/walletdb"
+)
+
+// CreditRecord contains metadata regarding a transaction credit for a known
+// transaction.  Further details may be looked up by indexing a wire.MsgTx.TxOut
+// with the Index field.
+type CreditRecord struct {
+	Index  uint32
+	Amount btcutil.Amount
+	Spent  bool
+	Change bool
+}
+
+// DebitRecord contains metadata regarding a transaction debit for a known
+// transaction.  Further details may be looked up by indexing a wire.MsgTx.TxIn
+// with the Index field.
+type DebitRecord struct {
+	Amount btcutil.Amount
+	Index  uint32
+}
+
+// TxDetails is intended to provide callers with access to rich details
+// regarding a relevant transaction and which inputs and outputs are credit or
+// debits.
+type TxDetails struct {
+	TxRecord
+	Block   BlockMeta
+	Credits []CreditRecord
+	Debits  []DebitRecord
+}
+
+// minedTxDetails fetches the TxDetails for the mined transaction with hash
+// txHash and the passed tx record key and value.
+func (s *Store) minedTxDetails(ns walletdb.Bucket, txHash *wire.ShaHash, recKey, recVal []byte) (*TxDetails, error) {
+	var details TxDetails
+
+	// Parse transaction record k/v, lookup the full block record for the
+	// block time, and read all matching credits, debits.
+	err := readRawTxRecord(txHash, recVal, &details.TxRecord)
+	if err != nil {
+		return nil, err
+	}
+	err = readRawTxRecordBlock(recKey, &details.Block.Block)
+	if err != nil {
+		return nil, err
+	}
+	details.Block.Time, err = fetchBlockTime(ns, details.Block.Height)
+	if err != nil {
+		return nil, err
+	}
+
+	credIter := makeCreditIterator(ns, recKey)
+	for credIter.next() {
+		if int(credIter.elem.Index) >= len(details.MsgTx.TxOut) {
+			str := "saved credit index exceeds number of outputs"
+			return nil, storeError(ErrData, str, nil)
+		}
+
+		// The credit iterator does not record whether this credit was
+		// spent by an unmined transaction, so check that here.
+		if !credIter.elem.Spent {
+			k := canonicalOutPoint(txHash, credIter.elem.Index)
+			spent := existsRawUnminedInput(ns, k) != nil
+			credIter.elem.Spent = spent
+		}
+		details.Credits = append(details.Credits, credIter.elem)
+	}
+	if credIter.err != nil {
+		return nil, credIter.err
+	}
+
+	debIter := makeDebitIterator(ns, recKey)
+	for debIter.next() {
+		if int(debIter.elem.Index) >= len(details.MsgTx.TxIn) {
+			str := "saved debit index exceeds number of inputs"
+			return nil, storeError(ErrData, str, nil)
+		}
+
+		details.Debits = append(details.Debits, debIter.elem)
+	}
+	return &details, debIter.err
+}
+
+// unminedTxDetails fetches the TxDetails for the unmined transaction with the
+// hash txHash and the passed unmined record value.
+func (s *Store) unminedTxDetails(ns walletdb.Bucket, txHash *wire.ShaHash, v []byte) (*TxDetails, error) {
+	details := TxDetails{
+		Block: BlockMeta{Block: Block{Height: -1}},
+	}
+	err := readRawTxRecord(txHash, v, &details.TxRecord)
+	if err != nil {
+		return nil, err
+	}
+
+	it := makeUnminedCreditIterator(ns, txHash)
+	for it.next() {
+		if int(it.elem.Index) >= len(details.MsgTx.TxOut) {
+			str := "saved credit index exceeds number of outputs"
+			return nil, storeError(ErrData, str, nil)
+		}
+
+		// Set the Spent field since this is not done by the iterator.
+		it.elem.Spent = existsRawUnminedInput(ns, it.ck) != nil
+		details.Credits = append(details.Credits, it.elem)
+	}
+	if it.err != nil {
+		return nil, it.err
+	}
+
+	// Debit records are not saved for unmined transactions.  Instead, they
+	// must be looked up for each transaction input manually.  There are two
+	// kinds of previous credits that may be debited by an unmined
+	// transaction: mined unspent outputs (which remain marked unspent even
+	// when spent by an unmined transaction), and credits from other unmined
+	// transactions.  Both situations must be considered.
+	for i, output := range details.MsgTx.TxIn {
+		opKey := canonicalOutPoint(&output.PreviousOutPoint.Hash,
+			output.PreviousOutPoint.Index)
+		credKey := existsRawUnspent(ns, opKey)
+		if credKey != nil {
+			v := existsRawCredit(ns, credKey)
+			amount, err := fetchRawCreditAmount(v)
+			if err != nil {
+				return nil, err
+			}
+
+			details.Debits = append(details.Debits, DebitRecord{
+				Amount: amount,
+				Index:  uint32(i),
+			})
+			continue
+		}
+
+		v := existsRawUnminedCredit(ns, opKey)
+		if v == nil {
+			continue
+		}
+
+		amount, err := fetchRawCreditAmount(v)
+		if err != nil {
+			return nil, err
+		}
+		details.Debits = append(details.Debits, DebitRecord{
+			Amount: amount,
+			Index:  uint32(i),
+		})
+	}
+
+	return &details, nil
+}
+
+// TxDetails looks up all recorded details regarding a transaction with some
+// hash.  In case of a hash collision, the most recent transaction with a
+// matching hash is returned.
+//
+// Not finding a transaction with this hash is not an error.  In this case,
+// a nil TxDetails is returned.
+func (s *Store) TxDetails(txHash *wire.ShaHash) (*TxDetails, error) {
+	var details *TxDetails
+	err := scopedView(s.namespace, func(ns walletdb.Bucket) error {
+		var err error
+
+		// First, check whether there exists an unmined transaction with this
+		// hash.  Use it if found.
+		v := existsRawUnmined(ns, txHash[:])
+		if v != nil {
+			details, err = s.unminedTxDetails(ns, txHash, v)
+			return err
+		}
+
+		// Otherwise, if there exists a mined transaction with this matching
+		// hash, skip over to the newest and begin fetching all details.
+		k, v := latestTxRecord(ns, txHash)
+		if v == nil {
+			// not found
+			return nil
+		}
+		details, err = s.minedTxDetails(ns, txHash, k, v)
+		return err
+	})
+	return details, err
+}
+
+// UniqueTxDetails looks up all recorded details for a transaction recorded
+// mined in some particular block, or an unmined transaction if block is nil.
+//
+// Not finding a transaction with this hash from this block is not an error.  In
+// this case, a nil TxDetails is returned.
+func (s *Store) UniqueTxDetails(txHash *wire.ShaHash, block *Block) (*TxDetails, error) {
+	var details *TxDetails
+	err := scopedView(s.namespace, func(ns walletdb.Bucket) error {
+		var err error
+		if block == nil {
+			v := existsRawUnmined(ns, txHash[:])
+			if v == nil {
+				return nil
+			}
+			details, err = s.unminedTxDetails(ns, txHash, v)
+			return err
+		}
+
+		k, v := existsTxRecord(ns, txHash, block)
+		if v == nil {
+			return nil
+		}
+		details, err = s.minedTxDetails(ns, txHash, k, v)
+		return err
+	})
+	return details, err
+}
+
+// rangeUnminedTransactions executes the function f with TxDetails for every
+// unmined transaction.  f is not executed if no unmined transactions exist.
+// Error returns from f (if any) are propigated to the caller.  Returns true
+// (signaling breaking out of a RangeTransactions) iff f executes and returns
+// true.
+func (s *Store) rangeUnminedTransactions(ns walletdb.Bucket, f func([]TxDetails) (bool, error)) (bool, error) {
+	var details []TxDetails
+	err := ns.Bucket(bucketUnmined).ForEach(func(k, v []byte) error {
+		if len(k) < 32 {
+			str := fmt.Sprintf("%s: short key (expected %d "+
+				"bytes, read %d)", bucketUnmined, 32, len(k))
+			return storeError(ErrData, str, nil)
+		}
+
+		var txHash wire.ShaHash
+		copy(txHash[:], k)
+		detail, err := s.unminedTxDetails(ns, &txHash, v)
+		if err != nil {
+			return err
+		}
+
+		// Because the key was created while foreach-ing over the
+		// bucket, it should be impossible for unminedTxDetails to ever
+		// successfully return a nil details struct.
+		details = append(details, *detail)
+		return nil
+	})
+	if err == nil && len(details) > 0 {
+		return f(details)
+	}
+	return false, err
+}
+
+// rangeBlockTransactions executes the function f with TxDetails for every block
+// between heights begin and end (reverse order when end > begin) until f
+// returns true, or the transactions from block is processed.  Returns true iff
+// f executes and returns true.
+func (s *Store) rangeBlockTransactions(ns walletdb.Bucket, begin, end int32, f func([]TxDetails) (bool, error)) (bool, error) {
+	// Mempool height is considered a high bound.
+	if begin < 0 {
+		begin = int32(^uint32(0) >> 1)
+	}
+	if end < 0 {
+		end = int32(^uint32(0) >> 1)
+	}
+
+	var blockIter blockIterator
+	var advance func(*blockIterator) bool
+	if begin < end {
+		// Iterate in forwards order
+		blockIter = makeBlockIterator(ns, begin)
+		advance = func(it *blockIterator) bool {
+			if !it.next() {
+				return false
+			}
+			return it.elem.Height <= end
+		}
+	} else {
+		// Iterate in backwards order, from begin -> end.
+		blockIter = makeBlockIterator(ns, begin)
+		advance = func(it *blockIterator) bool {
+			if !it.prev() {
+				return false
+			}
+			return end <= it.elem.Height
+		}
+	}
+
+	var details []TxDetails
+	for advance(&blockIter) {
+		block := &blockIter.elem
+
+		if cap(details) < len(block.transactions) {
+			details = make([]TxDetails, 0, len(block.transactions))
+		} else {
+			details = details[:0]
+		}
+
+		for _, txHash := range block.transactions {
+			k := keyTxRecord(&txHash, &block.Block)
+			v := existsRawTxRecord(ns, k)
+			if v == nil {
+				str := fmt.Sprintf("missing transaction %v for "+
+					"block %v", txHash, block.Height)
+				return false, storeError(ErrData, str, nil)
+			}
+			detail := TxDetails{
+				Block: BlockMeta{
+					Block: block.Block,
+					Time:  block.Time,
+				},
+			}
+			err := readRawTxRecord(&txHash, v, &detail.TxRecord)
+			if err != nil {
+				return false, err
+			}
+
+			credIter := makeCreditIterator(ns, k)
+			for credIter.next() {
+				if int(credIter.elem.Index) >= len(detail.MsgTx.TxOut) {
+					str := "saved credit index exceeds number of outputs"
+					return false, storeError(ErrData, str, nil)
+				}
+
+				// The credit iterator does not record whether
+				// this credit was spent by an unmined
+				// transaction, so check that here.
+				if !credIter.elem.Spent {
+					k := canonicalOutPoint(&txHash, credIter.elem.Index)
+					spent := existsRawUnminedInput(ns, k) != nil
+					credIter.elem.Spent = spent
+				}
+				detail.Credits = append(detail.Credits, credIter.elem)
+			}
+			if credIter.err != nil {
+				return false, credIter.err
+			}
+
+			debIter := makeDebitIterator(ns, k)
+			for debIter.next() {
+				if int(debIter.elem.Index) >= len(detail.MsgTx.TxIn) {
+					str := "saved debit index exceeds number of inputs"
+					return false, storeError(ErrData, str, nil)
+				}
+
+				detail.Debits = append(detail.Debits, debIter.elem)
+			}
+			if debIter.err != nil {
+				return false, debIter.err
+			}
+
+			details = append(details, detail)
+		}
+
+		// Every block record must have at least one transaction, so it
+		// is safe to call f.
+		brk, err := f(details)
+		if err != nil || brk {
+			return brk, err
+		}
+	}
+	return false, blockIter.err
+}
+
+// RangeTransactions runs the function f on all transaction details between
+// blocks on the best chain over the height range [begin,end].  The special
+// height -1 may be used to also include unmined transactions.  If the end
+// height comes before the begin height, blocks are iterated in reverse order
+// and unmined transactions (if any) are processed first.
+//
+// The function f may return an error which, if non-nil, is propagated to the
+// caller.  Additionally, a boolean return value allows exiting the function
+// early without reading any additional transactions early when true.
+//
+// All calls to f are guaranteed to be passed a slice with more than zero
+// elements.  The slice may be reused for multiple blocks, so it is not safe to
+// use it after the loop iteration it was acquired.
+func (s *Store) RangeTransactions(begin, end int32, f func([]TxDetails) (bool, error)) error {
+	return scopedView(s.namespace, func(ns walletdb.Bucket) error {
+		var addedUnmined bool
+		if begin < 0 {
+			brk, err := s.rangeUnminedTransactions(ns, f)
+			if err != nil || brk {
+				return err
+			}
+			addedUnmined = true
+		}
+
+		brk, err := s.rangeBlockTransactions(ns, begin, end, f)
+		if err == nil && !brk && !addedUnmined && end < 0 {
+			_, err = s.rangeUnminedTransactions(ns, f)
+		}
+		return err
+	})
+}
+
+// PreviousPkScripts returns a slice of previous output scripts for each credit
+// output this transaction record debits from.
+func (s *Store) PreviousPkScripts(rec *TxRecord, block *Block) ([][]byte, error) {
+	var pkScripts [][]byte
+	err := scopedView(s.namespace, func(ns walletdb.Bucket) error {
+		if block == nil {
+			for _, input := range rec.MsgTx.TxIn {
+				prevOut := &input.PreviousOutPoint
+
+				// Input may spend a previous unmined output, a
+				// mined output (which would still be marked
+				// unspent), or neither.
+
+				v := existsRawUnmined(ns, prevOut.Hash[:])
+				if v != nil {
+					// Ensure a credit exists for this
+					// unmined transaction before including
+					// the output script.
+					k := canonicalOutPoint(&prevOut.Hash, prevOut.Index)
+					if existsRawUnminedCredit(ns, k) == nil {
+						continue
+					}
+
+					pkScript, err := fetchRawTxRecordPkScript(
+						prevOut.Hash[:], v, prevOut.Index)
+					if err != nil {
+						return err
+					}
+					pkScripts = append(pkScripts, pkScript)
+					continue
+				}
+
+				_, credKey := existsUnspent(ns, prevOut)
+				if credKey != nil {
+					k := extractRawCreditTxRecordKey(credKey)
+					v = existsRawTxRecord(ns, k)
+					pkScript, err := fetchRawTxRecordPkScript(k, v,
+						prevOut.Index)
+					if err != nil {
+						return err
+					}
+					pkScripts = append(pkScripts, pkScript)
+				}
+			}
+			return nil
+		}
+
+		recKey := keyTxRecord(&rec.Hash, block)
+		it := makeDebitIterator(ns, recKey)
+		for it.next() {
+			credKey := extractRawDebitCreditKey(it.cv)
+			index := extractRawCreditIndex(credKey)
+			k := extractRawCreditTxRecordKey(credKey)
+			v := existsRawTxRecord(ns, k)
+			pkScript, err := fetchRawTxRecordPkScript(k, v, index)
+			if err != nil {
+				return err
+			}
+			pkScripts = append(pkScripts, pkScript)
+		}
+		return it.err
+	})
+	return pkScripts, err
+}

wtxmgr/query_test.go

@@ -0,0 +1,744 @@
+// Copyright (c) 2015 Conformal Systems LLC <info@conformal.com>
+//
+// Permission to use, copy, modify, and distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package wtxmgr_test
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	. "github.com/btcsuite/btcwallet/wtxmgr"
+)
+
+type queryState struct {
+	// slice items are ordered by height, mempool comes last.
+	blocks    [][]TxDetails
+	txDetails map[wire.ShaHash][]TxDetails
+}
+
+func newQueryState() *queryState {
+	return &queryState{
+		txDetails: make(map[wire.ShaHash][]TxDetails),
+	}
+}
+
+func (q *queryState) deepCopy() *queryState {
+	cpy := newQueryState()
+	for _, blockDetails := range q.blocks {
+		var cpyDetails []TxDetails
+		for _, detail := range blockDetails {
+			cpyDetails = append(cpyDetails, *deepCopyTxDetails(&detail))
+		}
+		cpy.blocks = append(cpy.blocks, cpyDetails)
+	}
+	cpy.txDetails = make(map[wire.ShaHash][]TxDetails)
+	for txHash, details := range q.txDetails {
+		detailsSlice := make([]TxDetails, len(details))
+		for i, detail := range details {
+			detailsSlice[i] = *deepCopyTxDetails(&detail)
+		}
+		cpy.txDetails[txHash] = detailsSlice
+	}
+	return cpy
+}
+
+func deepCopyTxDetails(d *TxDetails) *TxDetails {
+	cpy := *d
+	cpy.MsgTx = *d.MsgTx.Copy()
+	if cpy.SerializedTx != nil {
+		cpy.SerializedTx = make([]byte, len(cpy.SerializedTx))
+		copy(cpy.SerializedTx, d.SerializedTx)
+	}
+	cpy.Credits = make([]CreditRecord, len(d.Credits))
+	copy(cpy.Credits, d.Credits)
+	cpy.Debits = make([]DebitRecord, len(d.Debits))
+	copy(cpy.Debits, d.Debits)
+	return &cpy
+}
+
+func (q *queryState) compare(t *testing.T, s *Store, changeDesc string) {
+	defer func() {
+		if t.Failed() {
+			t.Fatalf("Store state queries failed after '%s'", changeDesc)
+		}
+	}()
+
+	fwdBlocks := q.blocks
+	revBlocks := make([][]TxDetails, len(q.blocks))
+	copy(revBlocks, q.blocks)
+	for i := 0; i < len(revBlocks)/2; i++ {
+		revBlocks[i], revBlocks[len(revBlocks)-1-i] = revBlocks[len(revBlocks)-1-i], revBlocks[i]
+	}
+	checkBlock := func(blocks [][]TxDetails) func([]TxDetails) (bool, error) {
+		return func(got []TxDetails) (bool, error) {
+			if len(fwdBlocks) == 0 {
+				return false, fmt.Errorf("entered range when no more details expected")
+			}
+			exp := blocks[0]
+			if len(got) != len(exp) {
+				return false, fmt.Errorf("got len(details)=%d in transaction range, expected %d", len(got), len(exp))
+			}
+			for i := range got {
+				equalTxDetails(t, &got[i], &exp[i])
+			}
+			if t.Failed() {
+				return false, fmt.Errorf("Failed comparing range of transaction details")
+			}
+			blocks = blocks[1:]
+			return false, nil
+		}
+	}
+	err := s.RangeTransactions(0, -1, checkBlock(fwdBlocks))
+	if err != nil {
+		t.Fatalf("Failed in RangeTransactions (forwards iteration): %v", err)
+	}
+	err = s.RangeTransactions(-1, 0, checkBlock(revBlocks))
+	if err != nil {
+		t.Fatalf("Failed in RangeTransactions (reverse iteration): %v", err)
+	}
+
+	for txHash, details := range q.txDetails {
+		for _, detail := range details {
+			blk := &detail.Block.Block
+			if blk.Height == -1 {
+				blk = nil
+			}
+			d, err := s.UniqueTxDetails(&txHash, blk)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if d == nil {
+				t.Errorf("Found no matching transaction at height %d", detail.Block.Height)
+				continue
+			}
+			equalTxDetails(t, d, &detail)
+		}
+		if t.Failed() {
+			t.Fatalf("Failed querying unique details regarding transaction %v", txHash)
+		}
+
+		// For the most recent tx with this hash, check that
+		// TxDetails (not looking up a tx at any particular
+		// height) matches the last.
+		detail := &details[len(details)-1]
+		d, err := s.TxDetails(&txHash)
+		if err != nil {
+			t.Fatal(err)
+		}
+		equalTxDetails(t, d, detail)
+		if t.Failed() {
+			t.Fatalf("Failed querying latest details regarding transaction %v", txHash)
+		}
+	}
+}
+
+func equalTxDetails(t *testing.T, got, exp *TxDetails) {
+	// Need to avoid using reflect.DeepEqual against slices, since it
+	// returns false for nil vs non-nil zero length slices.
+
+	equalTxs(t, &got.MsgTx, &exp.MsgTx)
+	if got.Hash != exp.Hash {
+		t.Errorf("Found mismatched hashes")
+		t.Errorf("Got: %v", got.Hash)
+		t.Errorf("Expected: %v", exp.Hash)
+	}
+	if got.Received != exp.Received {
+		t.Errorf("Found mismatched receive time")
+		t.Errorf("Got: %v", got.Received)
+		t.Errorf("Expected: %v", exp.Received)
+	}
+	if !bytes.Equal(got.SerializedTx, exp.SerializedTx) {
+		t.Errorf("Found mismatched serialized txs")
+		t.Errorf("Got: %x", got.SerializedTx)
+		t.Errorf("Expected: %x", exp.SerializedTx)
+	}
+	if got.Block != exp.Block {
+		t.Errorf("Found mismatched block meta")
+		t.Errorf("Got: %v", got.Block)
+		t.Errorf("Expected: %v", exp.Block)
+	}
+	if len(got.Credits) != len(exp.Credits) {
+		t.Errorf("Credit slice lengths differ: Got %d Expected %d", len(got.Credits), len(exp.Credits))
+	} else {
+		for i := range got.Credits {
+			if got.Credits[i] != exp.Credits[i] {
+				t.Errorf("Found mismatched Credit[%d]", i)
+				t.Errorf("Got: %v", got.Credits[i])
+				t.Errorf("Expected: %v", exp.Credits[i])
+			}
+		}
+	}
+	if len(got.Debits) != len(exp.Debits) {
+		t.Errorf("Debit slice lengths differ: Got %d Expected %d", len(got.Debits), len(exp.Debits))
+	} else {
+		for i := range got.Debits {
+			if got.Debits[i] != exp.Debits[i] {
+				t.Errorf("Found mismatched Debit[%d]", i)
+				t.Errorf("Got: %v", got.Debits[i])
+				t.Errorf("Expected: %v", exp.Debits[i])
+			}
+		}
+	}
+}
+
+func equalTxs(t *testing.T, got, exp *wire.MsgTx) {
+	var bufGot, bufExp bytes.Buffer
+	err := got.Serialize(&bufGot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = exp.Serialize(&bufExp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(bufGot.Bytes(), bufExp.Bytes()) {
+		t.Errorf("Found unexpected wire.MsgTx:")
+		t.Errorf("Got: %v", got)
+		t.Errorf("Expected: %v", exp)
+	}
+}
+
+// Returns time.Now() with seconds resolution, this is what Store saves.
+func timeNow() time.Time {
+	return time.Unix(time.Now().Unix(), 0)
+}
+
+// Returns a copy of a TxRecord without the serialized tx.
+func stripSerializedTx(rec *TxRecord) *TxRecord {
+	ret := *rec
+	ret.SerializedTx = nil
+	return &ret
+}
+
+func makeBlockMeta(height int32) BlockMeta {
+	if height == -1 {
+		return BlockMeta{Block: Block{Height: -1}}
+	}
+
+	b := BlockMeta{
+		Block: Block{Height: height},
+		Time:  timeNow(),
+	}
+	// Give it a fake block hash created from the height and time.
+	binary.LittleEndian.PutUint32(b.Hash[0:4], uint32(height))
+	binary.LittleEndian.PutUint64(b.Hash[4:12], uint64(b.Time.Unix()))
+	return b
+}
+
+func TestStoreQueries(t *testing.T) {
+	t.Parallel()
+
+	type queryTest struct {
+		desc    string
+		updates func() // Unwinds from t.Fatal if the update errors.
+		state   *queryState
+	}
+	var tests []queryTest
+
+	// Create the store and test initial state.
+	s, teardown, err := testStore()
+	defer teardown()
+	if err != nil {
+		t.Fatal(err)
+	}
+	lastState := newQueryState()
+	tests = append(tests, queryTest{
+		desc:    "initial store",
+		updates: func() {},
+		state:   lastState,
+	})
+
+	// simplify error handling
+	insertTx := func(rec *TxRecord, block *BlockMeta) {
+		err := s.InsertTx(rec, block)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	addCredit := func(s *Store, rec *TxRecord, block *BlockMeta, index uint32, change bool) {
+		err := s.AddCredit(rec, block, index, change)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	newTxRecordFromMsgTx := func(tx *wire.MsgTx, received time.Time) *TxRecord {
+		rec, err := NewTxRecordFromMsgTx(tx, received)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return rec
+	}
+	rollback := func(height int32) {
+		err := s.Rollback(height)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Insert an unmined transaction.  Mark no credits yet.
+	txA := spendOutput(&wire.ShaHash{}, 0, 100e8)
+	recA := newTxRecordFromMsgTx(txA, timeNow())
+	newState := lastState.deepCopy()
+	newState.blocks = [][]TxDetails{
+		{
+			{
+				TxRecord: *stripSerializedTx(recA),
+				Block:    BlockMeta{Block: Block{Height: -1}},
+			},
+		},
+	}
+	newState.txDetails[recA.Hash] = []TxDetails{
+		newState.blocks[0][0],
+	}
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "insert tx A unmined",
+		updates: func() { insertTx(recA, nil) },
+		state:   newState,
+	})
+
+	// Add txA:0 as a change credit.
+	newState = lastState.deepCopy()
+	newState.blocks[0][0].Credits = []CreditRecord{
+		{
+			Index:  0,
+			Amount: btcutil.Amount(recA.MsgTx.TxOut[0].Value),
+			Spent:  false,
+			Change: true,
+		},
+	}
+	newState.txDetails[recA.Hash][0].Credits = newState.blocks[0][0].Credits
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "mark unconfirmed txA:0 as credit",
+		updates: func() { addCredit(s, recA, nil, 0, true) },
+		state:   newState,
+	})
+
+	// Insert another unmined transaction which spends txA:0, splitting the
+	// amount into outputs of 40 and 60 BTC.
+	txB := spendOutput(&recA.Hash, 0, 40e8, 60e8)
+	recB := newTxRecordFromMsgTx(txB, timeNow())
+	newState = lastState.deepCopy()
+	newState.blocks[0][0].Credits[0].Spent = true
+	newState.blocks[0] = append(newState.blocks[0], TxDetails{
+		TxRecord: *stripSerializedTx(recB),
+		Block:    BlockMeta{Block: Block{Height: -1}},
+		Debits: []DebitRecord{
+			{
+				Amount: btcutil.Amount(recA.MsgTx.TxOut[0].Value),
+				Index:  0, // recB.MsgTx.TxIn index
+			},
+		},
+	})
+	newState.txDetails[recA.Hash][0].Credits[0].Spent = true
+	newState.txDetails[recB.Hash] = []TxDetails{newState.blocks[0][1]}
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "insert tx B unmined",
+		updates: func() { insertTx(recB, nil) },
+		state:   newState,
+	})
+	newState = lastState.deepCopy()
+	newState.blocks[0][1].Credits = []CreditRecord{
+		{
+			Index:  0,
+			Amount: btcutil.Amount(recB.MsgTx.TxOut[0].Value),
+			Spent:  false,
+			Change: false,
+		},
+	}
+	newState.txDetails[recB.Hash][0].Credits = newState.blocks[0][1].Credits
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "mark txB:0 as non-change credit",
+		updates: func() { addCredit(s, recB, nil, 0, false) },
+		state:   newState,
+	})
+
+	// Mine tx A at block 100.  Leave tx B unmined.
+	b100 := makeBlockMeta(100)
+	newState = lastState.deepCopy()
+	newState.blocks[0] = newState.blocks[0][:1]
+	newState.blocks[0][0].Block = b100
+	newState.blocks = append(newState.blocks, lastState.blocks[0][1:])
+	newState.txDetails[recA.Hash][0].Block = b100
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "mine tx A",
+		updates: func() { insertTx(recA, &b100) },
+		state:   newState,
+	})
+
+	// Mine tx B at block 101.
+	b101 := makeBlockMeta(101)
+	newState = lastState.deepCopy()
+	newState.blocks[1][0].Block = b101
+	newState.txDetails[recB.Hash][0].Block = b101
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "mine tx B",
+		updates: func() { insertTx(recB, &b101) },
+		state:   newState,
+	})
+
+	for _, tst := range tests {
+		tst.updates()
+		tst.state.compare(t, s, tst.desc)
+	}
+
+	// Run some additional query tests with the current store's state:
+	//   - Verify that querying for a transaction not in the store returns
+	//     nil without failure.
+	//   - Verify that querying for a unique transaction at the wrong block
+	//     returns nil without failure.
+	//   - Verify that breaking early on RangeTransactions stops further
+	//     iteration.
+
+	missingTx := spendOutput(&recB.Hash, 0, 40e8)
+	missingRec := newTxRecordFromMsgTx(missingTx, timeNow())
+	missingBlock := makeBlockMeta(102)
+	missingDetails, err := s.TxDetails(&missingRec.Hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if missingDetails != nil {
+		t.Errorf("Expected no details, found details for tx %v", missingDetails.Hash)
+	}
+	missingUniqueTests := []struct {
+		hash  *wire.ShaHash
+		block *Block
+	}{
+		{&missingRec.Hash, &b100.Block},
+		{&missingRec.Hash, &missingBlock.Block},
+		{&missingRec.Hash, nil},
+		{&recB.Hash, &b100.Block},
+		{&recB.Hash, &missingBlock.Block},
+		{&recB.Hash, nil},
+	}
+	for _, tst := range missingUniqueTests {
+		missingDetails, err = s.UniqueTxDetails(tst.hash, tst.block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if missingDetails != nil {
+			t.Errorf("Expected no details, found details for tx %v", missingDetails.Hash)
+		}
+	}
+
+	iterations := 0
+	err = s.RangeTransactions(0, -1, func([]TxDetails) (bool, error) {
+		iterations++
+		return true, nil
+	})
+	if iterations != 1 {
+		t.Errorf("RangeTransactions (forwards) ran func %d times", iterations)
+	}
+	iterations = 0
+	err = s.RangeTransactions(-1, 0, func([]TxDetails) (bool, error) {
+		iterations++
+		return true, nil
+	})
+	if iterations != 1 {
+		t.Errorf("RangeTransactions (reverse) ran func %d times", iterations)
+	}
+	// Make sure it also breaks early after one iteration through unmined transactions.
+	rollback(b101.Height)
+	iterations = 0
+	err = s.RangeTransactions(-1, 0, func([]TxDetails) (bool, error) {
+		iterations++
+		return true, nil
+	})
+	if iterations != 1 {
+		t.Errorf("RangeTransactions (reverse) ran func %d times", iterations)
+	}
+
+	// None of the above tests have tested RangeTransactions with multiple
+	// txs per block, so do that now.  Start by moving tx B to block 100
+	// (same block as tx A), and then rollback from block 100 onwards so
+	// both are unmined.
+	newState = lastState.deepCopy()
+	newState.blocks[0] = append(newState.blocks[0], newState.blocks[1]...)
+	newState.blocks[0][1].Block = b100
+	newState.blocks = newState.blocks[:1]
+	newState.txDetails[recB.Hash][0].Block = b100
+	lastState = newState
+	tests = append(tests[:0:0], queryTest{
+		desc:    "move tx B to block 100",
+		updates: func() { insertTx(recB, &b100) },
+		state:   newState,
+	})
+	newState = lastState.deepCopy()
+	newState.blocks[0][0].Block = makeBlockMeta(-1)
+	newState.blocks[0][1].Block = makeBlockMeta(-1)
+	newState.txDetails[recA.Hash][0].Block = makeBlockMeta(-1)
+	newState.txDetails[recB.Hash][0].Block = makeBlockMeta(-1)
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "rollback block 100",
+		updates: func() { rollback(b100.Height) },
+		state:   newState,
+	})
+
+	// None of the above tests have tested transactions with colliding
+	// hashes, so mine tx A in block 100, and then insert tx A again
+	// unmined.  Also mine tx A in block 101 (this moves it from unmined).
+	// This is a valid test because the store does not perform signature
+	// validation or keep a full utxo set, and duplicated transaction hashes
+	// from different blocks are allowed so long as all previous outputs are
+	// spent.
+	newState = lastState.deepCopy()
+	newState.blocks = append(newState.blocks, newState.blocks[0][1:])
+	newState.blocks[0] = newState.blocks[0][:1:1]
+	newState.blocks[0][0].Block = b100
+	newState.blocks[1] = []TxDetails{
+		{
+			TxRecord: *stripSerializedTx(recA),
+			Block:    makeBlockMeta(-1),
+		},
+		newState.blocks[1][0],
+	}
+	newState.txDetails[recA.Hash][0].Block = b100
+	newState.txDetails[recA.Hash] = append(newState.txDetails[recA.Hash], newState.blocks[1][0])
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "insert duplicate tx A",
+		updates: func() { insertTx(recA, &b100); insertTx(recA, nil) },
+		state:   newState,
+	})
+	newState = lastState.deepCopy()
+	newState.blocks = [][]TxDetails{
+		newState.blocks[0],
+		[]TxDetails{newState.blocks[1][0]},
+		[]TxDetails{newState.blocks[1][1]},
+	}
+	newState.blocks[1][0].Block = b101
+	newState.txDetails[recA.Hash][1].Block = b101
+	lastState = newState
+	tests = append(tests, queryTest{
+		desc:    "mine duplicate tx A",
+		updates: func() { insertTx(recA, &b101) },
+		state:   newState,
+	})
+
+	for _, tst := range tests {
+		tst.updates()
+		tst.state.compare(t, s, tst.desc)
+	}
+}
+
+func TestPreviousPkScripts(t *testing.T) {
+	t.Parallel()
+
+	s, teardown, err := testStore()
+	defer teardown()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Invalid scripts but sufficient for testing.
+	var (
+		scriptA0 = []byte("tx A output 0")
+		scriptA1 = []byte("tx A output 1")
+		scriptB0 = []byte("tx B output 0")
+		scriptB1 = []byte("tx B output 1")
+		scriptC0 = []byte("tx C output 0")
+		scriptC1 = []byte("tx C output 1")
+	)
+
+	// Create a transaction spending two prevous outputs and generating two
+	// new outputs the passed pkScipts.  Spends outputs 0 and 1 from prevHash.
+	buildTx := func(prevHash *wire.ShaHash, script0, script1 []byte) *wire.MsgTx {
+		return &wire.MsgTx{
+			TxIn: []*wire.TxIn{
+				&wire.TxIn{PreviousOutPoint: wire.OutPoint{*prevHash, 0}},
+				&wire.TxIn{PreviousOutPoint: wire.OutPoint{*prevHash, 1}},
+			},
+			TxOut: []*wire.TxOut{
+				&wire.TxOut{Value: 1e8, PkScript: script0},
+				&wire.TxOut{Value: 1e8, PkScript: script1},
+			},
+		}
+	}
+
+	newTxRecordFromMsgTx := func(tx *wire.MsgTx) *TxRecord {
+		rec, err := NewTxRecordFromMsgTx(tx, timeNow())
+		if err != nil {
+			t.Fatal(err)
+		}
+		return rec
+	}
+
+	// Create transactions with the fake output scripts.
+	var (
+		txA  = buildTx(&wire.ShaHash{}, scriptA0, scriptA1)
+		recA = newTxRecordFromMsgTx(txA)
+		txB  = buildTx(&recA.Hash, scriptB0, scriptB1)
+		recB = newTxRecordFromMsgTx(txB)
+		txC  = buildTx(&recB.Hash, scriptC0, scriptC1)
+		recC = newTxRecordFromMsgTx(txC)
+		txD  = buildTx(&recC.Hash, nil, nil)
+		recD = newTxRecordFromMsgTx(txD)
+	)
+
+	insertTx := func(rec *TxRecord, block *BlockMeta) {
+		err := s.InsertTx(rec, block)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	addCredit := func(rec *TxRecord, block *BlockMeta, index uint32) {
+		err := s.AddCredit(rec, block, index, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	type scriptTest struct {
+		rec     *TxRecord
+		block   *Block
+		scripts [][]byte
+	}
+	runTest := func(tst *scriptTest) {
+		scripts, err := s.PreviousPkScripts(tst.rec, tst.block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		height := int32(-1)
+		if tst.block != nil {
+			height = tst.block.Height
+		}
+		if len(scripts) != len(tst.scripts) {
+			t.Errorf("Transaction %v height %d: got len(scripts)=%d, expected %d",
+				tst.rec.Hash, height, len(scripts), len(tst.scripts))
+			return
+		}
+		for i := range scripts {
+			if !bytes.Equal(scripts[i], tst.scripts[i]) {
+				// Format scripts with %s since they are (should be) ascii.
+				t.Errorf("Transaction %v height %d script %d: got '%s' expected '%s'",
+					tst.rec.Hash, height, i, scripts[i], tst.scripts[i])
+			}
+		}
+	}
+
+	// Insert transactions A-C unmined, but mark no credits yet.  Until
+	// these are marked as credits, PreviousPkScripts should not return
+	// them.
+	insertTx(recA, nil)
+	insertTx(recB, nil)
+	insertTx(recC, nil)
+
+	b100 := makeBlockMeta(100)
+	b101 := makeBlockMeta(101)
+
+	tests := []scriptTest{
+		{recA, nil, nil},
+		{recA, &b100.Block, nil},
+		{recB, nil, nil},
+		{recB, &b100.Block, nil},
+		{recC, nil, nil},
+		{recC, &b100.Block, nil},
+	}
+	for _, tst := range tests {
+		runTest(&tst)
+	}
+	if t.Failed() {
+		t.Fatal("Failed after unmined tx inserts")
+	}
+
+	// Mark credits.  Tx C output 1 not marked as a credit: tx D will spend
+	// both later but when C is mined, output 1's script should not be
+	// returned.
+	addCredit(recA, nil, 0)
+	addCredit(recA, nil, 1)
+	addCredit(recB, nil, 0)
+	addCredit(recB, nil, 1)
+	addCredit(recC, nil, 0)
+	tests = []scriptTest{
+		{recA, nil, nil},
+		{recA, &b100.Block, nil},
+		{recB, nil, [][]byte{scriptA0, scriptA1}},
+		{recB, &b100.Block, nil},
+		{recC, nil, [][]byte{scriptB0, scriptB1}},
+		{recC, &b100.Block, nil},
+	}
+	for _, tst := range tests {
+		runTest(&tst)
+	}
+	if t.Failed() {
+		t.Fatal("Failed after marking unmined credits")
+	}
+
+	// Mine tx A in block 100.  Test results should be identical.
+	insertTx(recA, &b100)
+	for _, tst := range tests {
+		runTest(&tst)
+	}
+	if t.Failed() {
+		t.Fatal("Failed after mining tx A")
+	}
+
+	// Mine tx B in block 101.
+	insertTx(recB, &b101)
+	tests = []scriptTest{
+		{recA, nil, nil},
+		{recA, &b100.Block, nil},
+		{recB, nil, nil},
+		{recB, &b101.Block, [][]byte{scriptA0, scriptA1}},
+		{recC, nil, [][]byte{scriptB0, scriptB1}},
+		{recC, &b101.Block, nil},
+	}
+	for _, tst := range tests {
+		runTest(&tst)
+	}
+	if t.Failed() {
+		t.Fatal("Failed after mining tx B")
+	}
+
+	// Mine tx C in block 101 (same block as tx B) to test debits from the
+	// same block.
+	insertTx(recC, &b101)
+	tests = []scriptTest{
+		{recA, nil, nil},
+		{recA, &b100.Block, nil},
+		{recB, nil, nil},
+		{recB, &b101.Block, [][]byte{scriptA0, scriptA1}},
+		{recC, nil, nil},
+		{recC, &b101.Block, [][]byte{scriptB0, scriptB1}},
+	}
+	for _, tst := range tests {
+		runTest(&tst)
+	}
+	if t.Failed() {
+		t.Fatal("Failed after mining tx C")
+	}
+
+	// Insert tx D, which spends C:0 and C:1.  However, only C:0 is marked
+	// as a credit, and only that output script should be returned.
+	insertTx(recD, nil)
+	tests = append(tests, scriptTest{recD, nil, [][]byte{scriptC0}})
+	tests = append(tests, scriptTest{recD, &b101.Block, nil})
+	for _, tst := range tests {
+		runTest(&tst)
+	}
+	if t.Failed() {
+		t.Fatal("Failed after inserting tx D")
+	}
+}

wtxmgr/tx.go

@@ -0,0 +1,936 @@
+/*
+ * Copyright (c) 2013-2015 Conformal Systems LLC <info@conformal.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package wtxmgr
+
+import (
+	"bytes"
+	"time"
+
+	"github.com/btcsuite/btcd/blockchain"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/btcsuite/btcwallet/walletdb"
+)
+
+// Block contains the minimum amount of data to uniquely identify any block on
+// either the best or side chain.
+type Block struct {
+	Hash   wire.ShaHash
+	Height int32
+}
+
+// BlockMeta contains the unique identification for a block and any metadata
+// pertaining to the block.  At the moment, this additional metadata only
+// includes the block time from the block header.
+type BlockMeta struct {
+	Block
+	Time time.Time
+}
+
+// blockRecord is an in-memory representation of the block record saved in the
+// database.
+type blockRecord struct {
+	Block
+	Time         time.Time
+	transactions []wire.ShaHash
+}
+
+// incidence records the block hash and blockchain height of a mined transaction.
+// Since a transaction hash alone is not enough to uniquely identify a mined
+// transaction (duplicate transaction hashes are allowed), the incidence is used
+// instead.
+type incidence struct {
+	txHash wire.ShaHash
+	block  Block
+}
+
+// indexedIncidence records the transaction incidence and an input or output
+// index.
+type indexedIncidence struct {
+	incidence
+	index uint32
+}
+
+// debit records the debits a transaction record makes from previous wallet
+// transaction credits.
+type debit struct {
+	txHash wire.ShaHash
+	index  uint32
+	amount btcutil.Amount
+	spends indexedIncidence
+}
+
+// credit describes a transaction output which was or is spendable by wallet.
+type credit struct {
+	outPoint wire.OutPoint
+	block    Block
+	amount   btcutil.Amount
+	change   bool
+	spentBy  indexedIncidence // Index == ^uint32(0) if unspent
+}
+
+// TxRecord represents a transaction managed by the Store.
+type TxRecord struct {
+	MsgTx        wire.MsgTx
+	Hash         wire.ShaHash
+	Received     time.Time
+	SerializedTx []byte // Optional: may be nil
+}
+
+// NewTxRecord creates a new transaction record that may be inserted into the
+// store.  It uses memoization to save the transaction hash and the serialized
+// transaction.
+func NewTxRecord(serializedTx []byte, received time.Time) (*TxRecord, error) {
+	rec := &TxRecord{
+		Received:     received,
+		SerializedTx: serializedTx,
+	}
+	err := rec.MsgTx.Deserialize(bytes.NewReader(serializedTx))
+	if err != nil {
+		str := "failed to deserialize transaction"
+		return nil, storeError(ErrInput, str, err)
+	}
+	copy(rec.Hash[:], wire.DoubleSha256(serializedTx))
+	return rec, nil
+}
+
+// NewTxRecordFromMsgTx creates a new transaction record that may be inserted
+// into the store.
+func NewTxRecordFromMsgTx(msgTx *wire.MsgTx, received time.Time) (*TxRecord, error) {
+	buf := bytes.NewBuffer(make([]byte, 0, msgTx.SerializeSize()))
+	err := msgTx.Serialize(buf)
+	if err != nil {
+		str := "failed to serialize transaction"
+		return nil, storeError(ErrInput, str, err)
+	}
+	rec := &TxRecord{
+		MsgTx:        *msgTx,
+		Received:     received,
+		SerializedTx: buf.Bytes(),
+	}
+	copy(rec.Hash[:], wire.DoubleSha256(rec.SerializedTx))
+	return rec, nil
+}
+
+// Credit is the type representing a transaction output which was spent or
+// is still spendable by wallet.  A UTXO is an unspent Credit, but not all
+// Credits are UTXOs.
+type Credit struct {
+	wire.OutPoint
+	BlockMeta
+	Amount       btcutil.Amount
+	PkScript     []byte
+	Received     time.Time
+	FromCoinBase bool
+}
+
+// Store implements a transaction store for storing and managing wallet
+// transactions.
+type Store struct {
+	namespace walletdb.Namespace
+}
+
+// Open opens the wallet transaction store from a walletdb namespace.  If the
+// store does not exist, ErrNoExist is returned.  Existing stores will be
+// upgraded to new database formats as necessary.
+func Open(namespace walletdb.Namespace) (*Store, error) {
+	// Open the store, upgrading to the latest version as needed.
+	err := openStore(namespace)
+	if err != nil {
+		return nil, err
+	}
+	return &Store{namespace}, nil
+}
+
+// Create creates and opens a new persistent transaction store in the walletdb
+// namespace.  Creating the store when one already exists in this namespace will
+// error with ErrAlreadyExists.
+func Create(namespace walletdb.Namespace) (*Store, error) {
+	err := createStore(namespace)
+	if err != nil {
+		return nil, err
+	}
+	return &Store{namespace}, nil
+}
+
+// moveMinedTx moves a transaction record from the unmined buckets to block
+// buckets.
+func (s *Store) moveMinedTx(ns walletdb.Bucket, rec *TxRecord, recKey, recVal []byte, block *BlockMeta) error {
+	log.Infof("Marking unconfirmed transaction %v mined in block %d",
+		&rec.Hash, block.Height)
+
+	// Insert block record as needed.
+	blockKey, blockVal := existsBlockRecord(ns, block.Height)
+	var err error
+	if blockVal == nil {
+		blockVal = valueBlockRecord(block, &rec.Hash)
+	} else {
+		blockVal, err = appendRawBlockRecord(blockVal, &rec.Hash)
+		if err != nil {
+			return err
+		}
+	}
+	err = putRawBlockRecord(ns, blockKey, blockVal)
+	if err != nil {
+		return err
+	}
+
+	err = putRawTxRecord(ns, recKey, recVal)
+	if err != nil {
+		return err
+	}
+	minedBalance, err := fetchMinedBalance(ns)
+	if err != nil {
+		return err
+	}
+
+	// For all mined transactions with unspent credits spent by this
+	// transaction, mark each spent, remove from the unspents map, and
+	// insert a debit record for the spent credit.
+	debitIncidence := indexedIncidence{
+		incidence: incidence{txHash: rec.Hash, block: block.Block},
+		// index set for each rec input below.
+	}
+	for i, input := range rec.MsgTx.TxIn {
+		unspentKey, credKey := existsUnspent(ns, &input.PreviousOutPoint)
+		if credKey == nil {
+			continue
+		}
+		debitIncidence.index = uint32(i)
+		amt, err := spendCredit(ns, credKey, &debitIncidence)
+		if err != nil {
+			return err
+		}
+		minedBalance -= amt
+		err = deleteRawUnspent(ns, unspentKey)
+		if err != nil {
+			return err
+		}
+
+		err = putDebit(ns, &rec.Hash, uint32(i), amt, &block.Block, credKey)
+		if err != nil {
+			return err
+		}
+
+		err = deleteRawUnminedInput(ns, unspentKey)
+		if err != nil {
+			return err
+		}
+	}
+
+	// For each output of the record that is marked as a credit, if the
+	// output is marked as a credit by the unconfirmed store, remove the
+	// marker and mark the output as a credit in the db.
+	//
+	// Moved credits are added as unspents, even if there is another
+	// unconfirmed transaction which spends them.
+	cred := credit{
+		outPoint: wire.OutPoint{Hash: rec.Hash},
+		block:    block.Block,
+		spentBy:  indexedIncidence{index: ^uint32(0)},
+	}
+	it := makeUnminedCreditIterator(ns, &rec.Hash)
+	for it.next() {
+		// TODO: This should use the raw apis.  The credit value (it.cv)
+		// can be moved from unmined directly to the credits bucket.
+		// The key needs a modification to include the block
+		// height/hash.
+		index, err := fetchRawUnminedCreditIndex(it.ck)
+		if err != nil {
+			return err
+		}
+		amount, change, err := fetchRawUnminedCreditAmountChange(it.cv)
+		if err != nil {
+			return err
+		}
+		cred.outPoint.Index = index
+		cred.amount = amount
+		cred.change = change
+
+		err = it.delete()
+		if err != nil {
+			return err
+		}
+		err = putUnspentCredit(ns, &cred)
+		if err != nil {
+			return err
+		}
+		err = putUnspent(ns, &cred.outPoint, &block.Block)
+		if err != nil {
+			return err
+		}
+		minedBalance += amount
+	}
+	if it.err != nil {
+		return it.err
+	}
+
+	err = putMinedBalance(ns, minedBalance)
+	if err != nil {
+		return err
+	}
+
+	return deleteRawUnmined(ns, rec.Hash[:])
+}
+
+// InsertTx records a transaction as belonging to a wallet's transaction
+// history.  If block is nil, the transaction is considered unspent, and the
+// transaction's index must be unset.
+func (s *Store) InsertTx(rec *TxRecord, block *BlockMeta) error {
+	return scopedUpdate(s.namespace, func(ns walletdb.Bucket) error {
+		if block == nil {
+			return s.insertMemPoolTx(ns, rec)
+		}
+		return s.insertMinedTx(ns, rec, block)
+	})
+}
+
+// insertMinedTx inserts a new transaction record for a mined transaction into
+// the database.  It is expected that the exact transation does not already
+// exist in the unmined buckets, but unmined double spends (including mutations)
+// are removed.
+func (s *Store) insertMinedTx(ns walletdb.Bucket, rec *TxRecord, block *BlockMeta) error {
+	// If a transaction record for this tx hash and block already exist,
+	// there is nothing left to do.
+	k, v := existsTxRecord(ns, &rec.Hash, &block.Block)
+	if v != nil {
+		return nil
+	}
+
+	// If the exact tx (not a double spend) is already included but
+	// unconfirmed, move it to a block.
+	v = existsRawUnmined(ns, rec.Hash[:])
+	if v != nil {
+		return s.moveMinedTx(ns, rec, k, v, block)
+	}
+
+	// As there may be unconfirmed transactions that are invalidated by this
+	// transaction (either being duplicates, or double spends), remove them
+	// from the unconfirmed set.  This also handles removing unconfirmed
+	// transaction spend chains if any other unconfirmed transactions spend
+	// outputs of the removed double spend.
+	err := s.removeDoubleSpends(ns, rec)
+	if err != nil {
+		return err
+	}
+
+	// If a block record does not yet exist for any transactions from this
+	// block, insert the record.  Otherwise, update it by adding the
+	// transaction hash to the set of transactions from this block.
+	blockKey, blockValue := existsBlockRecord(ns, block.Height)
+	if blockValue == nil {
+		err = putBlockRecord(ns, block, &rec.Hash)
+	} else {
+		blockValue, err = appendRawBlockRecord(blockValue, &rec.Hash)
+		if err != nil {
+			return err
+		}
+		err = putRawBlockRecord(ns, blockKey, blockValue)
+	}
+	if err != nil {
+		return err
+	}
+
+	err = putTxRecord(ns, rec, &block.Block)
+	if err != nil {
+		return err
+	}
+
+	minedBalance, err := fetchMinedBalance(ns)
+	if err != nil {
+		return err
+	}
+
+	// Add a debit record for each unspent credit spent by this tx.
+	spender := indexedIncidence{
+		incidence: incidence{
+			txHash: rec.Hash,
+			block:  block.Block,
+		},
+		// index set for each iteration below
+	}
+	for i, input := range rec.MsgTx.TxIn {
+		unspentKey, credKey := existsUnspent(ns, &input.PreviousOutPoint)
+		if credKey == nil {
+			// Debits for unmined transactions are not explicitly
+			// tracked.  Instead, all previous outputs spent by any
+			// unmined transaction are added to a map for quick
+			// lookups when it must be checked whether a mined
+			// output is unspent or not.
+			//
+			// Tracking individual debits for unmined transactions
+			// could be added later to simplify (and increase
+			// performance of) determining some details that need
+			// the previous outputs (e.g. determining a fee), but at
+			// the moment that is not done (and a db lookup is used
+			// for those cases instead).  There is also a good
+			// chance that all unmined transaction handling will
+			// move entirely to the db rather than being handled in
+			// memory for atomicity reasons, so the simplist
+			// implementation is currently used.
+			continue
+		}
+		spender.index = uint32(i)
+		amt, err := spendCredit(ns, credKey, &spender)
+		if err != nil {
+			return err
+		}
+		err = putDebit(ns, &rec.Hash, uint32(i), amt, &block.Block,
+			credKey)
+		if err != nil {
+			return err
+		}
+
+		minedBalance -= amt
+
+		err = deleteRawUnspent(ns, unspentKey)
+		if err != nil {
+			return err
+		}
+	}
+
+	return putMinedBalance(ns, minedBalance)
+}
+
+// AddCredit marks a transaction record as containing a transaction output
+// spendable by wallet.  The output is added unspent, and is marked spent
+// when a new transaction spending the output is inserted into the store.
+//
+// TODO(jrick): This should not be necessary.  Instead, pass the indexes
+// that are known to contain credits when a transaction or merkleblock is
+// inserted into the store.
+func (s *Store) AddCredit(rec *TxRecord, block *BlockMeta, index uint32, change bool) error {
+	if int(index) >= len(rec.MsgTx.TxOut) {
+		str := "transaction output does not exist"
+		return storeError(ErrInput, str, nil)
+	}
+
+	return scopedUpdate(s.namespace, func(ns walletdb.Bucket) error {
+		return s.addCredit(ns, rec, block, index, change)
+	})
+}
+
+func (s *Store) addCredit(ns walletdb.Bucket, rec *TxRecord, block *BlockMeta, index uint32, change bool) error {
+	if block == nil {
+		k := canonicalOutPoint(&rec.Hash, index)
+		v := valueUnminedCredit(btcutil.Amount(rec.MsgTx.TxOut[index].Value), change)
+		return putRawUnminedCredit(ns, k, v)
+	}
+
+	k, v := existsCredit(ns, &rec.Hash, index, &block.Block)
+	if v != nil {
+		return nil
+	}
+
+	txOutAmt := btcutil.Amount(rec.MsgTx.TxOut[index].Value)
+	log.Debugf("Marking transaction %v output %d (%v) spendable",
+		rec.Hash, index, txOutAmt)
+
+	cred := credit{
+		outPoint: wire.OutPoint{
+			Hash:  rec.Hash,
+			Index: index,
+		},
+		block:   block.Block,
+		amount:  txOutAmt,
+		change:  change,
+		spentBy: indexedIncidence{index: ^uint32(0)},
+	}
+	v = valueUnspentCredit(&cred)
+	err := putRawCredit(ns, k, v)
+	if err != nil {
+		return err
+	}
+
+	minedBalance, err := fetchMinedBalance(ns)
+	if err != nil {
+		return err
+	}
+	err = putMinedBalance(ns, minedBalance+txOutAmt)
+	if err != nil {
+		return err
+	}
+
+	return putUnspent(ns, &cred.outPoint, &block.Block)
+}
+
+// Rollback removes all blocks at height onwards, moving any transactions within
+// each block to the unconfirmed pool.
+func (s *Store) Rollback(height int32) error {
+	return scopedUpdate(s.namespace, func(ns walletdb.Bucket) error {
+		return s.rollback(ns, height)
+	})
+}
+
+func (s *Store) rollback(ns walletdb.Bucket, height int32) error {
+	minedBalance, err := fetchMinedBalance(ns)
+	if err != nil {
+		return err
+	}
+
+	// Keep track of all credits that were removed from coinbase
+	// transactions.  After detaching all blocks, if any transaction record
+	// exists in unmined that spends these outputs, remove them and their
+	// spend chains.
+	//
+	// It is necessary to keep these in memory and fix the unmined
+	// transactions later since blocks are removed in increasing order.
+	var coinBaseCredits []wire.OutPoint
+
+	it := makeBlockIterator(ns, height)
+	for it.next() {
+		b := &it.elem
+
+		log.Infof("Rolling back %d transactions from block %v height %d",
+			len(b.transactions), b.Hash, b.Height)
+
+		for i := range b.transactions {
+			txHash := &b.transactions[i]
+
+			recKey := keyTxRecord(txHash, &b.Block)
+			recVal := existsRawTxRecord(ns, recKey)
+			var rec TxRecord
+			err = readRawTxRecord(txHash, recVal, &rec)
+			if err != nil {
+				return err
+			}
+
+			err = deleteTxRecord(ns, txHash, &b.Block)
+			if err != nil {
+				return err
+			}
+
+			// Handle coinbase transactions specially since they are
+			// not moved to the unconfirmed store.  A coinbase cannot
+			// contain any debits, but all credits should be removed
+			// and the mined balance decremented.
+			if blockchain.IsCoinBaseTx(&rec.MsgTx) {
+				op := wire.OutPoint{Hash: rec.Hash}
+				for i, output := range rec.MsgTx.TxOut {
+					k, v := existsCredit(ns, &rec.Hash,
+						uint32(i), &b.Block)
+					if v == nil {
+						continue
+					}
+					op.Index = uint32(i)
+
+					coinBaseCredits = append(coinBaseCredits, op)
+
+					unspentKey, credKey := existsUnspent(ns, &op)
+					if credKey != nil {
+						minedBalance -= btcutil.Amount(output.Value)
+						err = deleteRawUnspent(ns, unspentKey)
+						if err != nil {
+							return err
+						}
+					}
+					err = deleteRawCredit(ns, k)
+					if err != nil {
+						return err
+					}
+				}
+
+				continue
+			}
+
+			err = putRawUnmined(ns, txHash[:], recVal)
+			if err != nil {
+				return err
+			}
+
+			// For each debit recorded for this transaction, mark
+			// the credit it spends as unspent (as long as it still
+			// exists) and delete the debit.  The previous output is
+			// recorded in the unconfirmed store for every previous
+			// output, not just debits.
+			for i, input := range rec.MsgTx.TxIn {
+				prevOut := &input.PreviousOutPoint
+				prevOutKey := canonicalOutPoint(&prevOut.Hash,
+					prevOut.Index)
+				err = putRawUnminedInput(ns, prevOutKey, rec.Hash[:])
+				if err != nil {
+					return err
+				}
+
+				// If this input is a debit, remove the debit
+				// record and mark the credit that it spent as
+				// unspent, incrementing the mined balance.
+				debKey, credKey, err := existsDebit(ns,
+					&rec.Hash, uint32(i), &b.Block)
+				if err != nil {
+					return err
+				}
+				if debKey == nil {
+					continue
+				}
+
+				// unspendRawCredit does not error in case the
+				// no credit exists for this key, but this
+				// behavior is correct.  Since blocks are
+				// removed in increasing order, this credit
+				// may have already been removed from a
+				// previously removed transaction record in
+				// this rollback.
+				var amt btcutil.Amount
+				amt, err = unspendRawCredit(ns, credKey)
+				if err != nil {
+					return err
+				}
+				err = deleteRawDebit(ns, debKey)
+				if err != nil {
+					return err
+				}
+
+				// If the credit was previously removed in the
+				// rollback, the credit amount is zero.  Only
+				// mark the previously spent credit as unspent
+				// if it still exists.
+				if amt == 0 {
+					continue
+				}
+				unspentVal, err := fetchRawCreditUnspentValue(credKey)
+				if err != nil {
+					return err
+				}
+				minedBalance += amt
+				err = putRawUnspent(ns, prevOutKey, unspentVal)
+				if err != nil {
+					return err
+				}
+			}
+
+			// For each detached non-coinbase credit, move the
+			// credit output to unmined.  If the credit is marked
+			// unspent, it is removed from the utxo set and the
+			// mined balance is decremented.
+			//
+			// TODO: use a credit iterator
+			for i, output := range rec.MsgTx.TxOut {
+				k, v := existsCredit(ns, &rec.Hash, uint32(i),
+					&b.Block)
+				if v == nil {
+					continue
+				}
+
+				amt, change, err := fetchRawCreditAmountChange(v)
+				if err != nil {
+					return err
+				}
+				outPointKey := canonicalOutPoint(&rec.Hash, uint32(i))
+				unminedCredVal := valueUnminedCredit(amt, change)
+				err = putRawUnminedCredit(ns, outPointKey, unminedCredVal)
+				if err != nil {
+					return err
+				}
+
+				err = deleteRawCredit(ns, k)
+				if err != nil {
+					return err
+				}
+
+				credKey := existsRawUnspent(ns, outPointKey)
+				if credKey != nil {
+					minedBalance -= btcutil.Amount(output.Value)
+					err = deleteRawUnspent(ns, outPointKey)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+
+		err = it.delete()
+		if err != nil {
+			return err
+		}
+	}
+	if it.err != nil {
+		return it.err
+	}
+
+	for _, op := range coinBaseCredits {
+		opKey := canonicalOutPoint(&op.Hash, op.Index)
+		unminedKey := existsRawUnminedInput(ns, opKey)
+		if unminedKey != nil {
+			unminedVal := existsRawUnmined(ns, unminedKey)
+			var unminedRec TxRecord
+			copy(unminedRec.Hash[:], unminedKey) // Silly but need an array
+			err = readRawTxRecord(&unminedRec.Hash, unminedVal, &unminedRec)
+			if err != nil {
+				return err
+			}
+
+			log.Debugf("Transaction %v spends a removed coinbase "+
+				"output -- removing as well", unminedRec.Hash)
+			err = s.removeConflict(ns, &unminedRec)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return putMinedBalance(ns, minedBalance)
+}
+
+// UnspentOutputs returns all unspent received transaction outputs.
+// The order is undefined.
+func (s *Store) UnspentOutputs() ([]Credit, error) {
+	var credits []Credit
+	err := scopedView(s.namespace, func(ns walletdb.Bucket) error {
+		var err error
+		credits, err = s.unspentOutputs(ns)
+		return err
+	})
+	return credits, err
+}
+
+func (s *Store) unspentOutputs(ns walletdb.Bucket) ([]Credit, error) {
+	var unspent []Credit
+
+	var op wire.OutPoint
+	var block Block
+	err := ns.Bucket(bucketUnspent).ForEach(func(k, v []byte) error {
+		err := readCanonicalOutPoint(k, &op)
+		if err != nil {
+			return err
+		}
+		if existsRawUnminedInput(ns, k) != nil {
+			// Output is spent by an unmined transaction.
+			// Skip this k/v pair.
+			return nil
+		}
+		err = readUnspentBlock(v, &block)
+		if err != nil {
+			return err
+		}
+
+		blockTime, err := fetchBlockTime(ns, block.Height)
+		if err != nil {
+			return err
+		}
+		// TODO(jrick): reading the entire transaction should
+		// be avoidable.  Creating the credit only requires the
+		// output amount and pkScript.
+		rec, err := fetchTxRecord(ns, &op.Hash, &block)
+		if err != nil {
+			return err
+		}
+		txOut := rec.MsgTx.TxOut[op.Index]
+		cred := Credit{
+			OutPoint: op,
+			BlockMeta: BlockMeta{
+				Block: block,
+				Time:  blockTime,
+			},
+			Amount:       btcutil.Amount(txOut.Value),
+			PkScript:     txOut.PkScript,
+			Received:     rec.Received,
+			FromCoinBase: blockchain.IsCoinBaseTx(&rec.MsgTx),
+		}
+		unspent = append(unspent, cred)
+		return nil
+	})
+	if err != nil {
+		if _, ok := err.(Error); ok {
+			return nil, err
+		}
+		str := "failed iterating unspent bucket"
+		return nil, storeError(ErrDatabase, str, err)
+	}
+
+	err = ns.Bucket(bucketUnminedCredits).ForEach(func(k, v []byte) error {
+		if existsRawUnminedInput(ns, k) != nil {
+			// Output is spent by an unmined transaction.
+			// Skip to next unmined credit.
+			return nil
+		}
+
+		err := readCanonicalOutPoint(k, &op)
+		if err != nil {
+			return err
+		}
+
+		// TODO(jrick): Reading/parsing the entire transaction record
+		// just for the output amount and script can be avoided.
+		recVal := existsRawUnmined(ns, op.Hash[:])
+		var rec TxRecord
+		err = readRawTxRecord(&op.Hash, recVal, &rec)
+		if err != nil {
+			return err
+		}
+
+		txOut := rec.MsgTx.TxOut[op.Index]
+		cred := Credit{
+			OutPoint: op,
+			BlockMeta: BlockMeta{
+				Block: Block{Height: -1},
+			},
+			Amount:       btcutil.Amount(txOut.Value),
+			PkScript:     txOut.PkScript,
+			Received:     rec.Received,
+			FromCoinBase: blockchain.IsCoinBaseTx(&rec.MsgTx),
+		}
+		unspent = append(unspent, cred)
+		return nil
+	})
+	if err != nil {
+		if _, ok := err.(Error); ok {
+			return nil, err
+		}
+		str := "failed iterating unmined credits bucket"
+		return nil, storeError(ErrDatabase, str, err)
+	}
+
+	return unspent, nil
+}
+
+// Balance returns the spendable wallet balance (total value of all unspent
+// transaction outputs) given a minimum of minConf confirmations, calculated
+// at a current chain height of curHeight.  Coinbase outputs are only included
+// in the balance if maturity has been reached.
+//
+// Balance may return unexpected results if syncHeight is lower than the block
+// height of the most recent mined transaction in the store.
+func (s *Store) Balance(minConf, syncHeight int32) (btcutil.Amount, error) {
+	var amt btcutil.Amount
+	err := scopedView(s.namespace, func(ns walletdb.Bucket) error {
+		var err error
+		amt, err = s.balance(ns, minConf, syncHeight)
+		return err
+	})
+	return amt, err
+}
+
+func (s *Store) balance(ns walletdb.Bucket, minConf int32, syncHeight int32) (btcutil.Amount, error) {
+	bal, err := fetchMinedBalance(ns)
+	if err != nil {
+		return 0, err
+	}
+
+	// Subtract the balance for each credit that is spent by an unmined
+	// transaction.
+	var op wire.OutPoint
+	var block Block
+	err = ns.Bucket(bucketUnspent).ForEach(func(k, v []byte) error {
+		err := readCanonicalOutPoint(k, &op)
+		if err != nil {
+			return err
+		}
+		err = readUnspentBlock(v, &block)
+		if err != nil {
+			return err
+		}
+		if existsRawUnminedInput(ns, k) != nil {
+			_, v := existsCredit(ns, &op.Hash, op.Index, &block)
+			amt, err := fetchRawCreditAmount(v)
+			if err != nil {
+				return err
+			}
+			bal -= amt
+		}
+		return nil
+	})
+	if err != nil {
+		if _, ok := err.(Error); ok {
+			return 0, err
+		}
+		str := "failed iterating unspent outputs"
+		return 0, storeError(ErrDatabase, str, err)
+	}
+
+	// Decrement the balance for any unspent credit with less than
+	// minConf confirmations and any (unspent) immature coinbase credit.
+	stopConf := minConf
+	if blockchain.CoinbaseMaturity > stopConf {
+		stopConf = blockchain.CoinbaseMaturity
+	}
+	lastHeight := syncHeight - stopConf
+	blockIt := makeReverseBlockIterator(ns)
+	for blockIt.prev() {
+		block := &blockIt.elem
+
+		if block.Height < lastHeight {
+			break
+		}
+
+		for i := range block.transactions {
+			txHash := &block.transactions[i]
+			rec, err := fetchTxRecord(ns, txHash, &block.Block)
+			if err != nil {
+				return 0, err
+			}
+			numOuts := uint32(len(rec.MsgTx.TxOut))
+			for i := uint32(0); i < numOuts; i++ {
+				// Avoid double decrementing the credit amount
+				// if it was already removed for being spent by
+				// an unmined tx.
+				opKey := canonicalOutPoint(txHash, i)
+				if existsRawUnminedInput(ns, opKey) != nil {
+					continue
+				}
+
+				_, v := existsCredit(ns, txHash, i, &block.Block)
+				if v == nil {
+					continue
+				}
+				amt, spent, err := fetchRawCreditAmountSpent(v)
+				if err != nil {
+					return 0, err
+				}
+				if spent {
+					continue
+				}
+				confs := syncHeight - block.Height + 1
+				if confs < minConf || (blockchain.IsCoinBaseTx(&rec.MsgTx) &&
+					confs < blockchain.CoinbaseMaturity) {
+					bal -= amt
+				}
+			}
+		}
+	}
+	if blockIt.err != nil {
+		return 0, blockIt.err
+	}
+
+	// If unmined outputs are included, increment the balance for each
+	// output that is unspent.
+	if minConf == 0 {
+		err = ns.Bucket(bucketUnminedCredits).ForEach(func(k, v []byte) error {
+			if existsRawUnminedInput(ns, k) != nil {
+				// Output is spent by an unmined transaction.
+				// Skip to next unmined credit.
+				return nil
+			}
+
+			amount, err := fetchRawUnminedCreditAmount(v)
+			if err != nil {
+				return err
+			}
+			bal += amount
+			return nil
+		})
+		if err != nil {
+			if _, ok := err.(Error); ok {
+				return 0, err
+			}
+			str := "failed to iterate over unmined credits bucket"
+			return 0, storeError(ErrDatabase, str, err)
+		}
+	}
+
+	return bal, nil
+}

wtxmgr/unconfirmed.go

@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2013-2015 Conformal Systems LLC <info@conformal.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package wtxmgr
+
+import (
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcwallet/walletdb"
+)
+
+// insertMemPoolTx inserts the unmined transaction record.  It also marks
+// previous outputs referenced by the inputs as spent.
+func (s *Store) insertMemPoolTx(ns walletdb.Bucket, rec *TxRecord) error {
+	v := existsRawUnmined(ns, rec.Hash[:])
+	if v != nil {
+		// TODO: compare serialized txs to ensure this isn't a hash collision?
+		return nil
+	}
+
+	log.Infof("Inserting unconfirmed transaction %v", rec.Hash)
+	v, err := valueTxRecord(rec)
+	if err != nil {
+		return err
+	}
+	err = putRawUnmined(ns, rec.Hash[:], v)
+	if err != nil {
+		return err
+	}
+
+	for _, input := range rec.MsgTx.TxIn {
+		prevOut := &input.PreviousOutPoint
+		k := canonicalOutPoint(&prevOut.Hash, prevOut.Index)
+		err = putRawUnminedInput(ns, k, rec.Hash[:])
+		if err != nil {
+			return err
+		}
+	}
+
+	// TODO: increment credit amount for each credit (but those are unknown
+	// here currently).
+
+	return nil
+}
+
+// removeDoubleSpends checks for any unmined transactions which would introduce
+// a double spend if tx was added to the store (either as a confirmed or unmined
+// transaction).  Each conflicting transaction and all transactions which spend
+// it are recursively removed.
+func (s *Store) removeDoubleSpends(ns walletdb.Bucket, rec *TxRecord) error {
+	for _, input := range rec.MsgTx.TxIn {
+		prevOut := &input.PreviousOutPoint
+		prevOutKey := canonicalOutPoint(&prevOut.Hash, prevOut.Index)
+		doubleSpendHash := existsRawUnminedInput(ns, prevOutKey)
+		if doubleSpendHash != nil {
+			var doubleSpend TxRecord
+			doubleSpendVal := existsRawUnmined(ns, doubleSpendHash)
+			copy(doubleSpend.Hash[:], doubleSpendHash) // Silly but need an array
+			err := readRawTxRecord(&doubleSpend.Hash, doubleSpendVal,
+				&doubleSpend)
+			if err != nil {
+				return err
+			}
+
+			log.Debugf("Removing double spending transaction %v",
+				doubleSpend.Hash)
+			err = s.removeConflict(ns, &doubleSpend)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// removeConflict removes an unmined transaction record and all spend chains
+// deriving from it from the store.  This is designed to remove transactions
+// that would otherwise result in double spend conflicts if left in the store,
+// and to remove transactions that spend coinbase transactions on reorgs.
+func (s *Store) removeConflict(ns walletdb.Bucket, rec *TxRecord) error {
+	// For each potential credit for this record, each spender (if any) must
+	// be recursively removed as well.  Once the spenders are removed, the
+	// credit is deleted.
+	numOuts := uint32(len(rec.MsgTx.TxOut))
+	for i := uint32(0); i < numOuts; i++ {
+		k := canonicalOutPoint(&rec.Hash, i)
+		spenderHash := existsRawUnminedInput(ns, k)
+		if spenderHash != nil {
+			var spender TxRecord
+			spenderVal := existsRawUnmined(ns, spenderHash)
+			copy(spender.Hash[:], spenderHash) // Silly but need an array
+			err := readRawTxRecord(&spender.Hash, spenderVal, &spender)
+			if err != nil {
+				return err
+			}
+
+			log.Debugf("Transaction %v is part of a removed conflict "+
+				"chain -- removing as well", spender.Hash)
+			err = s.removeConflict(ns, &spender)
+			if err != nil {
+				return err
+			}
+		}
+		err := deleteRawUnminedCredit(ns, k)
+		if err != nil {
+			return err
+		}
+	}
+
+	// If this tx spends any previous credits (either mined or unmined), set
+	// each unspent.  Mined transactions are only marked spent by having the
+	// output in the unmined inputs bucket.
+	for _, input := range rec.MsgTx.TxIn {
+		prevOut := &input.PreviousOutPoint
+		k := canonicalOutPoint(&prevOut.Hash, prevOut.Index)
+		err := deleteRawUnminedInput(ns, k)
+		if err != nil {
+			return err
+		}
+	}
+
+	return deleteRawUnmined(ns, rec.Hash[:])
+}
+
+// UnminedTxs returns the underlying transactions for all unmined transactions
+// which are not known to have been mined in a block.
+func (s *Store) UnminedTxs() ([]*wire.MsgTx, error) {
+	var txs []*wire.MsgTx
+	err := scopedView(s.namespace, func(ns walletdb.Bucket) error {
+		var err error
+		txs, err = s.unminedTxs(ns)
+		return err
+	})
+	return txs, err
+}
+
+func (s *Store) unminedTxs(ns walletdb.Bucket) ([]*wire.MsgTx, error) {
+	var unmined []*wire.MsgTx
+	err := ns.Bucket(bucketUnmined).ForEach(func(k, v []byte) error {
+		// TODO: Parsing transactions from the db may be a little
+		// expensive.  It's possible the caller only wants the
+		// serialized transactions.
+		var txHash wire.ShaHash
+		err := readRawUnminedHash(k, &txHash)
+		if err != nil {
+			return err
+		}
+
+		var rec TxRecord
+		err = readRawTxRecord(&txHash, v, &rec)
+		if err != nil {
+			return err
+		}
+
+		tx := rec.MsgTx
+		unmined = append(unmined, &tx)
+		return nil
+	})
+	return unmined, err
+}