diff --git a/wtxmgr/README.md b/wtxmgr/README.md new file mode 100644 index 0000000..1cd2676 --- /dev/null +++ b/wtxmgr/README.md @@ -0,0 +1,45 @@ +wtxmgr +====== + +[![Build Status](https://travis-ci.org/btcsuite/btcwallet.png?branch=master)] +(https://travis-ci.org/btcsuite/btcwallet) + +Package wtxmgr provides storage and spend tracking of wallet transactions and +their relevant input and outputs. + +## Feature overview + +- Storage for relevant wallet transactions +- Ability to mark outputs as controlled by wallet +- Unspent transaction output index +- Balance tracking +- Automatic spend tracking for transaction inserts and removals +- Double spend detection and correction after blockchain reorgs +- Scalable design: + - Utilizes similar prefixes to allow cursor iteration over relevant transaction + inputs and outputs + - Programmatically detectable errors, including encapsulation of errors from + packages it relies on + - Operates under its own walletdb namespace + +## Documentation + +[![GoDoc](https://godoc.org/github.com/btcsuite/btcwallet/wtxmgr?status.png)] +(http://godoc.org/github.com/btcsuite/btcwallet/wtxmgr) + +Full `go doc` style documentation for the project can be viewed online without +installing this package by using the GoDoc site here: +http://godoc.org/github.com/btcsuite/btcwallet/wtxmgr + +You can also view the documentation locally once the package is installed with +the `godoc` tool by running `godoc -http=":6060"` and pointing your browser to +http://localhost:6060/pkg/github.com/btcsuite/btcwallet/wtxmgr + +## Installation + +```bash +$ go get github.com/btcsuite/btcwallet/wtxmgr +``` + +Package wtxmgr is licensed under the [copyfree](http://copyfree.org) ISC +License. diff --git a/wtxmgr/db.go b/wtxmgr/db.go new file mode 100644 index 0000000..2809501 --- /dev/null +++ b/wtxmgr/db.go @@ -0,0 +1,1395 @@ +/* + * Copyright (c) 2015 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package wtxmgr + +import ( + "bytes" + "encoding/binary" + "fmt" + "time" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwallet/walletdb" +) + +// Naming +// +// The following variables are commonly used in this file and given +// reserved names: +// +// ns: The namespace bucket for this package +// b: The primary bucket being operated on +// k: A single bucket key +// v: A single bucket value +// c: A bucket cursor +// ck: The current cursor key +// cv: The current cursor value +// +// Functions use the naming scheme `Op[Raw]Type[Field]`, which performs the +// operation `Op` on the type `Type`, optionally dealing with raw keys and +// values if `Raw` is used. Fetch and extract operations may only need to read +// some portion of a key or value, in which case `Field` describes the component +// being returned. The following operations are used: +// +// key: return a db key for some data +// value: return a db value for some data +// put: insert or replace a value into a bucket +// fetch: read and return a value +// read: read a value into an out parameter +// exists: return the raw (nil if not found) value for some data +// delete: remove a k/v pair +// extract: perform an unchecked slice to extract a key or value +// +// Other operations which are specific to the types being operated on +// should be explained in a comment. + +// Big endian is the preferred byte order, due to cursor scans over integer +// keys iterating in order. +var byteOrder = binary.BigEndian + +// Database versions. Versions start at 1 and increment for each database +// change. +const ( + // LatestVersion is the most recent store version. + LatestVersion = 1 +) + +// This package makes assumptions that the width of a wire.ShaHash is always 32 +// bytes. If this is ever changed (unlikely for bitcoin, possible for alts), +// offsets have to be rewritten. Use a compile-time assertion that this +// assumption holds true. +var _ [32]byte = wire.ShaHash{} + +// Bucket names +var ( + bucketBlocks = []byte("b") + bucketTxRecords = []byte("t") + bucketCredits = []byte("c") + bucketUnspent = []byte("u") + bucketDebits = []byte("d") + bucketUnmined = []byte("m") + bucketUnminedCredits = []byte("mc") + bucketUnminedInputs = []byte("mi") +) + +// Root (namespace) bucket keys +var ( + rootCreateDate = []byte("date") + rootVersion = []byte("vers") + rootMinedBalance = []byte("bal") +) + +// The root bucket's mined balance k/v pair records the total balance for all +// unspent credits from mined transactions. This includes immature outputs, and +// outputs spent by mempool transactions, which must be considered when +// returning the actual balance for a given number of block confirmations. The +// value is the amount serialized as a uint64. + +func fetchMinedBalance(ns walletdb.Bucket) (btcutil.Amount, error) { + v := ns.Get(rootMinedBalance) + if len(v) != 8 { + str := fmt.Sprintf("balance: short read (expected 8 bytes, "+ + "read %v)", len(v)) + return 0, storeError(ErrData, str, nil) + } + return btcutil.Amount(byteOrder.Uint64(v)), nil +} + +func putMinedBalance(ns walletdb.Bucket, amt btcutil.Amount) error { + v := make([]byte, 8) + byteOrder.PutUint64(v, uint64(amt)) + err := ns.Put(rootMinedBalance, v) + if err != nil { + str := "failed to put balance" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// Several data structures are given canonical serialization formats as either +// keys or values. These common formats allow keys and values to be reused +// across different buckets. +// +// The canonical outpoint serialization format is: +// +// [0:32] Trasaction hash (32 bytes) +// [32:36] Output index (4 bytes) +// +// The canonical transaction hash serialization is simply the hash. + +func canonicalOutPoint(txHash *wire.ShaHash, index uint32) []byte { + k := make([]byte, 36) + copy(k, txHash[:]) + byteOrder.PutUint32(k[32:36], index) + return k +} + +func readCanonicalOutPoint(k []byte, op *wire.OutPoint) error { + if len(k) < 36 { + str := "short canonical outpoint" + return storeError(ErrData, str, nil) + } + copy(op.Hash[:], k) + op.Index = byteOrder.Uint32(k[32:36]) + return nil +} + +// Details regarding blocks are saved as k/v pairs in the blocks bucket. +// blockRecords are keyed by their height. The value is serialized as such: +// +// [0:32] Hash (32 bytes) +// [32:40] Unix time (8 bytes) +// [40:44] Number of transaction hashes (4 bytes) +// [44:] For each transaction hash: +// Hash (32 bytes) + +func keyBlockRecord(height int32) []byte { + k := make([]byte, 4) + byteOrder.PutUint32(k, uint32(height)) + return k +} + +func valueBlockRecord(block *BlockMeta, txHash *wire.ShaHash) []byte { + v := make([]byte, 76) + copy(v, block.Hash[:]) + byteOrder.PutUint64(v[32:40], uint64(block.Time.Unix())) + byteOrder.PutUint32(v[40:44], 1) + copy(v[44:76], txHash[:]) + return v +} + +// appendRawBlockRecord returns a new block record value with a transaction +// hash appended to the end and an incremented number of transactions. +func appendRawBlockRecord(v []byte, txHash *wire.ShaHash) ([]byte, error) { + if len(v) < 44 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketBlocks, 44, len(v)) + return nil, storeError(ErrData, str, nil) + } + newv := append(v[:len(v):len(v)], txHash[:]...) + n := byteOrder.Uint32(newv[40:44]) + byteOrder.PutUint32(newv[40:44], n+1) + return newv, nil +} + +func putRawBlockRecord(ns walletdb.Bucket, k, v []byte) error { + err := ns.Bucket(bucketBlocks).Put(k, v) + if err != nil { + str := "failed to store block" + return storeError(ErrDatabase, str, err) + } + return nil +} + +func putBlockRecord(ns walletdb.Bucket, block *BlockMeta, txHash *wire.ShaHash) error { + k := keyBlockRecord(block.Height) + v := valueBlockRecord(block, txHash) + return putRawBlockRecord(ns, k, v) +} + +func fetchBlockTime(ns walletdb.Bucket, height int32) (time.Time, error) { + k := keyBlockRecord(height) + v := ns.Bucket(bucketBlocks).Get(k) + if len(v) < 44 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketBlocks, 44, len(v)) + return time.Time{}, storeError(ErrData, str, nil) + } + return time.Unix(int64(byteOrder.Uint64(v[32:40])), 0), nil +} + +func existsBlockRecord(ns walletdb.Bucket, height int32) (k, v []byte) { + k = keyBlockRecord(height) + v = ns.Bucket(bucketBlocks).Get(k) + return +} + +func readRawBlockRecord(k, v []byte, block *blockRecord) error { + if len(k) < 4 { + str := fmt.Sprintf("%s: short key (expected %d bytes, read %d)", + bucketBlocks, 4, len(k)) + return storeError(ErrData, str, nil) + } + if len(v) < 44 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketBlocks, 44, len(v)) + return storeError(ErrData, str, nil) + } + numTransactions := int(byteOrder.Uint32(v[40:44])) + expectedLen := 44 + wire.HashSize*numTransactions + if len(v) < expectedLen { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketBlocks, expectedLen, len(v)) + return storeError(ErrData, str, nil) + } + + block.Height = int32(byteOrder.Uint32(k)) + copy(block.Hash[:], v) + block.Time = time.Unix(int64(byteOrder.Uint64(v[32:40])), 0) + block.transactions = make([]wire.ShaHash, numTransactions) + off := 44 + for i := range block.transactions { + copy(block.transactions[i][:], v[off:]) + off += wire.HashSize + } + + return nil +} + +type blockIterator struct { + c walletdb.Cursor + seek []byte + ck []byte + cv []byte + elem blockRecord + err error +} + +func makeBlockIterator(ns walletdb.Bucket, height int32) blockIterator { + seek := make([]byte, 4) + byteOrder.PutUint32(seek, uint32(height)) + c := ns.Bucket(bucketBlocks).Cursor() + return blockIterator{c: c, seek: seek} +} + +// Works just like makeBlockIterator but will initially position the cursor at +// the last k/v pair. Use this with blockIterator.prev. +func makeReverseBlockIterator(ns walletdb.Bucket) blockIterator { + seek := make([]byte, 4) + byteOrder.PutUint32(seek, ^uint32(0)) + c := ns.Bucket(bucketBlocks).Cursor() + return blockIterator{c: c, seek: seek} +} + +func (it *blockIterator) next() bool { + if it.c == nil { + return false + } + + if it.ck == nil { + it.ck, it.cv = it.c.Seek(it.seek) + } else { + it.ck, it.cv = it.c.Next() + } + if it.ck == nil { + it.c = nil + return false + } + + err := readRawBlockRecord(it.ck, it.cv, &it.elem) + if err != nil { + it.c = nil + it.err = err + return false + } + + return true +} + +func (it *blockIterator) prev() bool { + if it.c == nil { + return false + } + + if it.ck == nil { + it.ck, it.cv = it.c.Seek(it.seek) + // Seek positions the cursor at the next k/v pair if one with + // this prefix was not found. If this happened (the prefixes + // won't match in this case) move the cursor backward. + // + // This technically does not correct for multiple keys with + // matching prefixes by moving the cursor to the last matching + // key, but this doesn't need to be considered when dealing with + // block records since the key (and seek prefix) is just the + // block height. + if !bytes.HasPrefix(it.ck, it.seek) { + it.ck, it.cv = it.c.Prev() + } + } else { + it.ck, it.cv = it.c.Prev() + } + if it.ck == nil { + it.c = nil + return false + } + + err := readRawBlockRecord(it.ck, it.cv, &it.elem) + if err != nil { + it.c = nil + it.err = err + return false + } + + return true +} + +func (it *blockIterator) delete() error { + err := it.c.Delete() + if err != nil { + str := "failed to delete block record" + storeError(ErrDatabase, str, err) + } + return nil +} + +// Transaction records are keyed as such: +// +// [0:32] Transaction hash (32 bytes) +// [32:36] Block height (4 bytes) +// [36:68] Block hash (32 bytes) +// +// The leading transaction hash allows to prefix filter for all records with +// a matching hash. The block height and hash records a particular incidence +// of the transaction in the blockchain. +// +// The record value is serialized as such: +// +// [0:8] Received time (8 bytes) +// [8:] Serialized transaction (varies) + +func keyTxRecord(txHash *wire.ShaHash, block *Block) []byte { + k := make([]byte, 68) + copy(k, txHash[:]) + byteOrder.PutUint32(k[32:36], uint32(block.Height)) + copy(k[36:68], block.Hash[:]) + return k +} + +func valueTxRecord(rec *TxRecord) ([]byte, error) { + var v []byte + if rec.SerializedTx == nil { + txSize := rec.MsgTx.SerializeSize() + v = make([]byte, 8, 8+txSize) + err := rec.MsgTx.Serialize(bytes.NewBuffer(v[8:])) + if err != nil { + str := fmt.Sprintf("unable to serialize transaction %v", rec.Hash) + return nil, storeError(ErrInput, str, err) + } + v = v[:cap(v)] + } else { + v = make([]byte, 8+len(rec.SerializedTx)) + copy(v[8:], rec.SerializedTx) + } + byteOrder.PutUint64(v, uint64(rec.Received.Unix())) + return v, nil +} + +func putTxRecord(ns walletdb.Bucket, rec *TxRecord, block *Block) error { + k := keyTxRecord(&rec.Hash, block) + v, err := valueTxRecord(rec) + if err != nil { + return err + } + err = ns.Bucket(bucketTxRecords).Put(k, v) + if err != nil { + str := fmt.Sprintf("%s: put failed for %v", bucketTxRecords, rec.Hash) + return storeError(ErrDatabase, str, err) + } + return nil +} + +func putRawTxRecord(ns walletdb.Bucket, k, v []byte) error { + err := ns.Bucket(bucketTxRecords).Put(k, v) + if err != nil { + str := fmt.Sprintf("%s: put failed", bucketTxRecords) + return storeError(ErrDatabase, str, err) + } + return nil +} + +func readRawTxRecord(txHash *wire.ShaHash, v []byte, rec *TxRecord) error { + if len(v) < 8 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketTxRecords, 8, len(v)) + return storeError(ErrData, str, nil) + } + rec.Hash = *txHash + rec.Received = time.Unix(int64(byteOrder.Uint64(v)), 0) + err := rec.MsgTx.Deserialize(bytes.NewReader(v[8:])) + if err != nil { + str := fmt.Sprintf("%s: failed to deserialize transaction %v", + bucketTxRecords, txHash) + return storeError(ErrData, str, err) + } + return nil +} + +func readRawTxRecordBlock(k []byte, block *Block) error { + if len(k) < 68 { + str := fmt.Sprintf("%s: short key (expected %d bytes, read %d)", + bucketTxRecords, 68, len(k)) + return storeError(ErrData, str, nil) + } + block.Height = int32(byteOrder.Uint32(k[32:36])) + copy(block.Hash[:], k[36:68]) + return nil +} + +func fetchTxRecord(ns walletdb.Bucket, txHash *wire.ShaHash, block *Block) (*TxRecord, error) { + k := keyTxRecord(txHash, block) + v := ns.Bucket(bucketTxRecords).Get(k) + + rec := new(TxRecord) + err := readRawTxRecord(txHash, v, rec) + return rec, err +} + +// TODO: This reads more than necessary. Pass the pkscript location instead to +// avoid the wire.MsgTx deserialization. +func fetchRawTxRecordPkScript(k, v []byte, index uint32) ([]byte, error) { + var rec TxRecord + copy(rec.Hash[:], k) // Silly but need an array + err := readRawTxRecord(&rec.Hash, v, &rec) + if err != nil { + return nil, err + } + if int(index) >= len(rec.MsgTx.TxOut) { + str := "missing transaction output for credit index" + return nil, storeError(ErrData, str, nil) + } + return rec.MsgTx.TxOut[index].PkScript, nil +} + +func existsTxRecord(ns walletdb.Bucket, txHash *wire.ShaHash, block *Block) (k, v []byte) { + k = keyTxRecord(txHash, block) + v = ns.Bucket(bucketTxRecords).Get(k) + return +} + +func existsRawTxRecord(ns walletdb.Bucket, k []byte) (v []byte) { + return ns.Bucket(bucketTxRecords).Get(k) +} + +func deleteTxRecord(ns walletdb.Bucket, txHash *wire.ShaHash, block *Block) error { + k := keyTxRecord(txHash, block) + return ns.Bucket(bucketTxRecords).Delete(k) +} + +// latestTxRecord searches for the newest recorded mined transaction record with +// a matching hash. In case of a hash collision, the record from the newest +// block is returned. Returns (nil, nil) if no matching transactions are found. +func latestTxRecord(ns walletdb.Bucket, txHash *wire.ShaHash) (k, v []byte) { + prefix := txHash[:] + c := ns.Bucket(bucketTxRecords).Cursor() + ck, cv := c.Seek(prefix) + var lastKey, lastVal []byte + for bytes.HasPrefix(ck, prefix) { + lastKey, lastVal = ck, cv + ck, cv = c.Next() + } + return lastKey, lastVal +} + +// All transaction credits (outputs) are keyed as such: +// +// [0:32] Transaction hash (32 bytes) +// [32:36] Block height (4 bytes) +// [36:68] Block hash (32 bytes) +// [68:72] Output index (4 bytes) +// +// The first 68 bytes match the key for the transaction record and may be used +// as a prefix filter to iterate through all credits in order. +// +// The credit value is serialized as such: +// +// [0:8] Amount (8 bytes) +// [8] Flags (1 byte) +// 0x01: Spent +// 0x02: Change +// [9:81] OPTIONAL Debit bucket key (72 bytes) +// [9:41] Spender transaction hash (32 bytes) +// [41:45] Spender block height (4 bytes) +// [45:77] Spender block hash (32 bytes) +// [77:81] Spender transaction input index (4 bytes) +// +// The optional debits key is only included if the credit is spent by another +// mined debit. + +func keyCredit(txHash *wire.ShaHash, index uint32, block *Block) []byte { + k := make([]byte, 72) + copy(k, txHash[:]) + byteOrder.PutUint32(k[32:36], uint32(block.Height)) + copy(k[36:68], block.Hash[:]) + byteOrder.PutUint32(k[68:72], index) + return k +} + +// valueUnspentCredit creates a new credit value for an unspent credit. All +// credits are created unspent, and are only marked spent later, so there is no +// value function to create either spent or unspent credits. +func valueUnspentCredit(cred *credit) []byte { + v := make([]byte, 9) + byteOrder.PutUint64(v, uint64(cred.amount)) + if cred.change { + v[8] |= 1 << 1 + } + return v +} + +func putRawCredit(ns walletdb.Bucket, k, v []byte) error { + err := ns.Bucket(bucketCredits).Put(k, v) + if err != nil { + str := "failed to put credit" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// putUnspentCredit puts a credit record for an unspent credit. It may only be +// used when the credit is already know to be unspent, or spent by an +// unconfirmed transaction. +func putUnspentCredit(ns walletdb.Bucket, cred *credit) error { + k := keyCredit(&cred.outPoint.Hash, cred.outPoint.Index, &cred.block) + v := valueUnspentCredit(cred) + return putRawCredit(ns, k, v) +} + +func extractRawCreditTxRecordKey(k []byte) []byte { + return k[0:68] +} + +func extractRawCreditIndex(k []byte) uint32 { + return byteOrder.Uint32(k[68:72]) +} + +// fetchRawCreditAmount returns the amount of the credit. +func fetchRawCreditAmount(v []byte) (btcutil.Amount, error) { + if len(v) < 9 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketCredits, 9, len(v)) + return 0, storeError(ErrData, str, nil) + } + return btcutil.Amount(byteOrder.Uint64(v)), nil +} + +// fetchRawCreditAmountSpent returns the amount of the credit and whether the +// credit is spent. +func fetchRawCreditAmountSpent(v []byte) (btcutil.Amount, bool, error) { + if len(v) < 9 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketCredits, 9, len(v)) + return 0, false, storeError(ErrData, str, nil) + } + return btcutil.Amount(byteOrder.Uint64(v)), v[8]&(1<<0) != 0, nil +} + +// fetchRawCreditAmountChange returns the amount of the credit and whether the +// credit is marked as change. +func fetchRawCreditAmountChange(v []byte) (btcutil.Amount, bool, error) { + if len(v) < 9 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketCredits, 9, len(v)) + return 0, false, storeError(ErrData, str, nil) + } + return btcutil.Amount(byteOrder.Uint64(v)), v[8]&(1<<1) != 0, nil +} + +// fetchRawCreditUnspentValue returns the unspent value for a raw credit key. +// This may be used to mark a credit as unspent. +func fetchRawCreditUnspentValue(k []byte) ([]byte, error) { + if len(k) < 72 { + str := fmt.Sprintf("%s: short key (expected %d bytes, read %d)", + bucketCredits, 72, len(k)) + return nil, storeError(ErrData, str, nil) + } + return k[32:68], nil +} + +// spendRawCredit marks the credit with a given key as mined at some particular +// block as spent by the input at some transaction incidence. The debited +// amount is returned. +func spendCredit(ns walletdb.Bucket, k []byte, spender *indexedIncidence) (btcutil.Amount, error) { + v := ns.Bucket(bucketCredits).Get(k) + newv := make([]byte, 81) + copy(newv, v) + v = newv + v[8] |= 1 << 0 + copy(v[9:41], spender.txHash[:]) + byteOrder.PutUint32(v[41:45], uint32(spender.block.Height)) + copy(v[45:77], spender.block.Hash[:]) + byteOrder.PutUint32(v[77:81], spender.index) + + return btcutil.Amount(byteOrder.Uint64(v[0:8])), putRawCredit(ns, k, v) +} + +// unspendRawCredit rewrites the credit for the given key as unspent. The +// output amount of the credit is returned. It returns without error if no +// credit exists for the key. +func unspendRawCredit(ns walletdb.Bucket, k []byte) (btcutil.Amount, error) { + b := ns.Bucket(bucketCredits) + v := b.Get(k) + if v == nil { + return 0, nil + } + newv := make([]byte, 9) + copy(newv, v) + newv[8] &^= 1 << 0 + + err := b.Put(k, newv) + if err != nil { + str := "failed to put credit" + return 0, storeError(ErrDatabase, str, err) + } + return btcutil.Amount(byteOrder.Uint64(v[0:8])), nil +} + +func existsCredit(ns walletdb.Bucket, txHash *wire.ShaHash, index uint32, block *Block) (k, v []byte) { + k = keyCredit(txHash, index, block) + v = ns.Bucket(bucketCredits).Get(k) + return +} + +func existsRawCredit(ns walletdb.Bucket, k []byte) []byte { + return ns.Bucket(bucketCredits).Get(k) +} + +func deleteRawCredit(ns walletdb.Bucket, k []byte) error { + err := ns.Bucket(bucketCredits).Delete(k) + if err != nil { + str := "failed to delete credit" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// creditIterator allows for in-order iteration of all credit records for a +// mined transaction. +// +// Example usage: +// +// prefix := keyTxRecord(txHash, block) +// it := makeCreditIterator(ns, prefix) +// for it.next() { +// // Use it.elem +// // If necessary, read additional details from it.ck, it.cv +// } +// if it.err != nil { +// // Handle error +// } +// +// The elem's Spent field is not set to true if the credit is spent by an +// unmined transaction. To check for this case: +// +// k := canonicalOutPoint(&txHash, it.elem.Index) +// it.elem.Spent = existsRawUnminedInput(ns, k) != nil +type creditIterator struct { + c walletdb.Cursor // Set to nil after final iteration + prefix []byte + ck []byte + cv []byte + elem CreditRecord + err error +} + +func makeCreditIterator(ns walletdb.Bucket, prefix []byte) creditIterator { + c := ns.Bucket(bucketCredits).Cursor() + return creditIterator{c: c, prefix: prefix} +} + +func (it *creditIterator) readElem() error { + if len(it.ck) < 72 { + str := fmt.Sprintf("%s: short key (expected %d bytes, read %d)", + bucketCredits, 72, len(it.ck)) + return storeError(ErrData, str, nil) + } + if len(it.cv) < 9 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketCredits, 9, len(it.cv)) + return storeError(ErrData, str, nil) + } + it.elem.Index = byteOrder.Uint32(it.ck[68:72]) + it.elem.Amount = btcutil.Amount(byteOrder.Uint64(it.cv)) + it.elem.Spent = it.cv[8]&(1<<0) != 0 + it.elem.Change = it.cv[8]&(1<<1) != 0 + return nil +} + +func (it *creditIterator) next() bool { + if it.c == nil { + return false + } + + if it.ck == nil { + it.ck, it.cv = it.c.Seek(it.prefix) + } else { + it.ck, it.cv = it.c.Next() + } + if !bytes.HasPrefix(it.ck, it.prefix) { + it.c = nil + return false + } + + err := it.readElem() + if err != nil { + it.err = err + return false + } + return true +} + +// The unspent index records all outpoints for mined credits which are not spent +// by any other mined transaction records (but may be spent by a mempool +// transaction). +// +// Keys are use the canonical outpoint serialization: +// +// [0:32] Transaction hash (32 bytes) +// [32:36] Output index (4 bytes) +// +// Values are serialized as such: +// +// [0:4] Block height (4 bytes) +// [4:36] Block hash (32 bytes) + +func valueUnspent(block *Block) []byte { + v := make([]byte, 36) + byteOrder.PutUint32(v, uint32(block.Height)) + copy(v[4:36], block.Hash[:]) + return v +} + +func putUnspent(ns walletdb.Bucket, outPoint *wire.OutPoint, block *Block) error { + k := canonicalOutPoint(&outPoint.Hash, outPoint.Index) + v := valueUnspent(block) + err := ns.Bucket(bucketUnspent).Put(k, v) + if err != nil { + str := "cannot put unspent" + return storeError(ErrDatabase, str, err) + } + return nil +} + +func putRawUnspent(ns walletdb.Bucket, k, v []byte) error { + err := ns.Bucket(bucketUnspent).Put(k, v) + if err != nil { + str := "cannot put unspent" + return storeError(ErrDatabase, str, err) + } + return nil +} + +func readUnspentBlock(v []byte, block *Block) error { + if len(v) < 36 { + str := "short unspent value" + return storeError(ErrData, str, nil) + } + block.Height = int32(byteOrder.Uint32(v)) + copy(block.Hash[:], v[4:36]) + return nil +} + +// existsUnspent returns the key for the unspent output and the cooresponding +// key for the credits bucket. If there is no unspent output recorded, the +// credit key is nil. +func existsUnspent(ns walletdb.Bucket, outPoint *wire.OutPoint) (k, credKey []byte) { + k = canonicalOutPoint(&outPoint.Hash, outPoint.Index) + credKey = existsRawUnspent(ns, k) + return k, credKey +} + +// existsRawUnspent returns the credit key if there exists an output recorded +// for the raw unspent key. It returns nil if the k/v pair does not exist. +func existsRawUnspent(ns walletdb.Bucket, k []byte) (credKey []byte) { + if len(k) < 36 { + return nil + } + v := ns.Bucket(bucketUnspent).Get(k) + if len(v) < 36 { + return nil + } + credKey = make([]byte, 72) + copy(credKey, k[:32]) + copy(credKey[32:68], v) + copy(credKey[68:72], k[32:36]) + return credKey +} + +func deleteRawUnspent(ns walletdb.Bucket, k []byte) error { + err := ns.Bucket(bucketUnspent).Delete(k) + if err != nil { + str := "failed to delete unspent" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// All transaction debits (inputs which spend credits) are keyed as such: +// +// [0:32] Transaction hash (32 bytes) +// [32:36] Block height (4 bytes) +// [36:68] Block hash (32 bytes) +// [68:72] Input index (4 bytes) +// +// The first 68 bytes match the key for the transaction record and may be used +// as a prefix filter to iterate through all debits in order. +// +// The debit value is serialized as such: +// +// [0:8] Amount (8 bytes) +// [8:80] Credits bucket key (72 bytes) +// [8:40] Transaction hash (32 bytes) +// [40:44] Block height (4 bytes) +// [44:76] Block hash (32 bytes) +// [76:80] Output index (4 bytes) + +func keyDebit(txHash *wire.ShaHash, index uint32, block *Block) []byte { + k := make([]byte, 72) + copy(k, txHash[:]) + byteOrder.PutUint32(k[32:36], uint32(block.Height)) + copy(k[36:68], block.Hash[:]) + byteOrder.PutUint32(k[68:72], index) + return k +} + +func putDebit(ns walletdb.Bucket, txHash *wire.ShaHash, index uint32, amount btcutil.Amount, block *Block, credKey []byte) error { + k := keyDebit(txHash, index, block) + + v := make([]byte, 80) + byteOrder.PutUint64(v, uint64(amount)) + copy(v[8:80], credKey) + + err := ns.Bucket(bucketDebits).Put(k, v) + if err != nil { + str := fmt.Sprintf("failed to update debit %s input %d", + txHash, index) + return storeError(ErrDatabase, str, err) + } + return nil +} + +func extractRawDebitCreditKey(v []byte) []byte { + return v[8:80] +} + +// existsDebit checks for the existance of a debit. If found, the debit and +// previous credit keys are returned. If the debit does not exist, both keys +// are nil. +func existsDebit(ns walletdb.Bucket, txHash *wire.ShaHash, index uint32, block *Block) (k, credKey []byte, err error) { + k = keyDebit(txHash, index, block) + v := ns.Bucket(bucketDebits).Get(k) + if v == nil { + return nil, nil, nil + } + if len(v) < 80 { + str := fmt.Sprintf("%s: short read (expected 80 bytes, read %v)", + bucketDebits, len(v)) + return nil, nil, storeError(ErrData, str, nil) + } + return k, v[8:80], nil +} + +func deleteRawDebit(ns walletdb.Bucket, k []byte) error { + err := ns.Bucket(bucketDebits).Delete(k) + if err != nil { + str := "failed to delete debit" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// debitIterator allows for in-order iteration of all debit records for a +// mined transaction. +// +// Example usage: +// +// prefix := keyTxRecord(txHash, block) +// it := makeDebitIterator(ns, prefix) +// for it.next() { +// // Use it.elem +// // If necessary, read additional details from it.ck, it.cv +// } +// if it.err != nil { +// // Handle error +// } +type debitIterator struct { + c walletdb.Cursor // Set to nil after final iteration + prefix []byte + ck []byte + cv []byte + elem DebitRecord + err error +} + +func makeDebitIterator(ns walletdb.Bucket, prefix []byte) debitIterator { + c := ns.Bucket(bucketDebits).Cursor() + return debitIterator{c: c, prefix: prefix} +} + +func (it *debitIterator) readElem() error { + if len(it.ck) < 72 { + str := fmt.Sprintf("%s: short key (expected %d bytes, read %d)", + bucketDebits, 72, len(it.ck)) + return storeError(ErrData, str, nil) + } + if len(it.cv) < 80 { + str := fmt.Sprintf("%s: short read (expected %d bytes, read %d)", + bucketDebits, 80, len(it.cv)) + return storeError(ErrData, str, nil) + } + it.elem.Index = byteOrder.Uint32(it.ck[68:72]) + it.elem.Amount = btcutil.Amount(byteOrder.Uint64(it.cv)) + return nil +} + +func (it *debitIterator) next() bool { + if it.c == nil { + return false + } + + if it.ck == nil { + it.ck, it.cv = it.c.Seek(it.prefix) + } else { + it.ck, it.cv = it.c.Next() + } + if !bytes.HasPrefix(it.ck, it.prefix) { + it.c = nil + return false + } + + err := it.readElem() + if err != nil { + it.err = err + return false + } + return true +} + +// All unmined transactions are saved in the unmined bucket keyed by the +// transaction hash. The value matches that of mined transaction records: +// +// [0:8] Received time (8 bytes) +// [8:] Serialized transaction (varies) + +func putRawUnmined(ns walletdb.Bucket, k, v []byte) error { + err := ns.Bucket(bucketUnmined).Put(k, v) + if err != nil { + str := "failed to put unmined record" + return storeError(ErrDatabase, str, err) + } + return nil +} + +func readRawUnminedHash(k []byte, txHash *wire.ShaHash) error { + if len(k) < 32 { + str := "short unmined key" + return storeError(ErrData, str, nil) + } + copy(txHash[:], k) + return nil +} + +func existsRawUnmined(ns walletdb.Bucket, k []byte) (v []byte) { + return ns.Bucket(bucketUnmined).Get(k) +} + +func deleteRawUnmined(ns walletdb.Bucket, k []byte) error { + err := ns.Bucket(bucketUnmined).Delete(k) + if err != nil { + str := "failed to delete unmined record" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// Unmined transaction credits use the canonical serialization format: +// +// [0:32] Transaction hash (32 bytes) +// [32:36] Output index (4 bytes) +// +// The value matches the format used by mined credits, but the spent flag is +// never set and the optional debit record is never included. The simplified +// format is thus: +// +// [0:8] Amount (8 bytes) +// [8] Flags (1 byte) +// 0x02: Change + +func valueUnminedCredit(amount btcutil.Amount, change bool) []byte { + v := make([]byte, 9) + byteOrder.PutUint64(v, uint64(amount)) + if change { + v[8] = 1 << 1 + } + return v +} + +func putRawUnminedCredit(ns walletdb.Bucket, k, v []byte) error { + err := ns.Bucket(bucketUnminedCredits).Put(k, v) + if err != nil { + str := "cannot put unmined credit" + return storeError(ErrDatabase, str, err) + } + return nil +} + +func fetchRawUnminedCreditIndex(k []byte) (uint32, error) { + if len(k) < 36 { + str := "short unmined credit key" + return 0, storeError(ErrData, str, nil) + } + return byteOrder.Uint32(k[32:36]), nil +} + +func fetchRawUnminedCreditAmount(v []byte) (btcutil.Amount, error) { + if len(v) < 9 { + str := "short unmined credit value" + return 0, storeError(ErrData, str, nil) + } + return btcutil.Amount(byteOrder.Uint64(v)), nil +} + +func fetchRawUnminedCreditAmountChange(v []byte) (btcutil.Amount, bool, error) { + if len(v) < 9 { + str := "short unmined credit value" + return 0, false, storeError(ErrData, str, nil) + } + amt := btcutil.Amount(byteOrder.Uint64(v)) + change := v[8]&(1<<1) != 0 + return amt, change, nil +} + +func existsRawUnminedCredit(ns walletdb.Bucket, k []byte) []byte { + return ns.Bucket(bucketUnminedCredits).Get(k) +} + +func deleteRawUnminedCredit(ns walletdb.Bucket, k []byte) error { + err := ns.Bucket(bucketUnminedCredits).Delete(k) + if err != nil { + str := "failed to delete unmined credit" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// unminedCreditIterator allows for cursor iteration over all credits, in order, +// from a single unmined transaction. +// +// Example usage: +// +// it := makeUnminedCreditIterator(ns, txHash) +// for it.next() { +// // Use it.elem, it.ck and it.cv +// // Optionally, use it.delete() to remove this k/v pair +// } +// if it.err != nil { +// // Handle error +// } +// +// The spentness of the credit is not looked up for performance reasons (because +// for unspent credits, it requires another lookup in another bucket). If this +// is needed, it may be checked like this: +// +// spent := existsRawUnminedInput(ns, it.ck) != nil +type unminedCreditIterator struct { + c walletdb.Cursor + prefix []byte + ck []byte + cv []byte + elem CreditRecord + err error +} + +func makeUnminedCreditIterator(ns walletdb.Bucket, txHash *wire.ShaHash) unminedCreditIterator { + c := ns.Bucket(bucketUnminedCredits).Cursor() + return unminedCreditIterator{c: c, prefix: txHash[:]} +} + +func (it *unminedCreditIterator) readElem() error { + index, err := fetchRawUnminedCreditIndex(it.ck) + if err != nil { + return err + } + amount, change, err := fetchRawUnminedCreditAmountChange(it.cv) + if err != nil { + return err + } + + it.elem.Index = index + it.elem.Amount = amount + it.elem.Change = change + // Spent intentionally not set + + return nil +} + +func (it *unminedCreditIterator) next() bool { + if it.c == nil { + return false + } + + if it.ck == nil { + it.ck, it.cv = it.c.Seek(it.prefix) + } else { + it.ck, it.cv = it.c.Next() + } + if !bytes.HasPrefix(it.ck, it.prefix) { + it.c = nil + return false + } + + err := it.readElem() + if err != nil { + it.err = err + return false + } + return true +} + +func (it *unminedCreditIterator) delete() error { + err := it.c.Delete() + if err != nil { + str := "failed to delete unmined credit" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// Outpoints spent by unmined transactions are saved in the unmined inputs +// bucket. This bucket maps between each previous output spent, for both mined +// and unmined transactions, to the hash of the unmined transaction. +// +// The key is serialized as such: +// +// [0:32] Transaction hash (32 bytes) +// [32:36] Output index (4 bytes) +// +// The value is serialized as such: +// +// [0:32] Transaction hash (32 bytes) + +func putRawUnminedInput(ns walletdb.Bucket, k, v []byte) error { + err := ns.Bucket(bucketUnminedInputs).Put(k, v) + if err != nil { + str := "failed to put unmined input" + return storeError(ErrDatabase, str, err) + } + return nil +} + +func existsRawUnminedInput(ns walletdb.Bucket, k []byte) (v []byte) { + return ns.Bucket(bucketUnminedInputs).Get(k) +} + +func deleteRawUnminedInput(ns walletdb.Bucket, k []byte) error { + err := ns.Bucket(bucketUnminedInputs).Delete(k) + if err != nil { + str := "failed to delete unmined input" + return storeError(ErrDatabase, str, err) + } + return nil +} + +// openStore opens an existing transaction store from the passed namespace. If +// necessary, an already existing store is upgraded to newer db format. +func openStore(namespace walletdb.Namespace) error { + var version uint32 + err := scopedView(namespace, func(ns walletdb.Bucket) error { + // Verify a store already exists and upgrade as necessary. + v := ns.Get(rootVersion) + if len(v) != 4 { + return nil + } + version = byteOrder.Uint32(v) + return nil + }) + if err != nil { + const desc = "failed to open existing store" + if serr, ok := err.(Error); ok { + serr.Desc = desc + ": " + serr.Desc + return serr + } + return storeError(ErrDatabase, desc, err) + } + + // The initial version is one. If no store exists and no version was + // saved, this variable will be zero. + if version == 0 { + str := "no transaction store exists in namespace" + return storeError(ErrNoExists, str, nil) + } + + // Cannot continue if the saved database is too new for this software. + // This probably indicates an outdated binary. + if version > LatestVersion { + str := fmt.Sprintf("recorded version %d is newer that latest "+ + "understood version %d", version, LatestVersion) + return storeError(ErrUnknownVersion, str, nil) + } + + // Upgrade the tx store as needed, one version at a time, until + // LatestVersion is reached. Versions are not skipped when performing + // database upgrades, and each upgrade is done in its own transaction. + // + // No upgrades yet. + //if version < LatestVersion { + // err := scopedUpdate(namespace, func(ns walletdb.Bucket) error { + // }) + // if err != nil { + // // Handle err + // } + //} + + return nil +} + +// createStore creates the tx store (with the latest db version) in the passed +// namespace. If a store already exists, ErrAlreadyExists is returned. +func createStore(namespace walletdb.Namespace) error { + // Initialize the buckets and root bucket fields as needed. + err := scopedUpdate(namespace, func(ns walletdb.Bucket) error { + // Ensure that nothing currently exists in the namespace bucket. + ck, cv := ns.Cursor().First() + if ck != nil || cv != nil { + const str = "namespace is not empty" + return storeError(ErrAlreadyExists, str, nil) + } + + // Write the latest store version. + v := make([]byte, 4) + byteOrder.PutUint32(v, LatestVersion) + err := ns.Put(rootVersion, v) + if err != nil { + str := "failed to store latest database version" + return storeError(ErrDatabase, str, err) + } + + // Save the creation date of the store. + v = make([]byte, 8) + byteOrder.PutUint64(v, uint64(time.Now().Unix())) + err = ns.Put(rootCreateDate, v) + if err != nil { + str := "failed to store database creation time" + return storeError(ErrDatabase, str, err) + } + + // Write a zero balance. + v = make([]byte, 8) + err = ns.Put(rootMinedBalance, v) + if err != nil { + str := "failed to write zero balance" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketBlocks) + if err != nil { + str := "failed to create blocks bucket" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketTxRecords) + if err != nil { + str := "failed to create tx records bucket" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketCredits) + if err != nil { + str := "failed to create credits bucket" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketDebits) + if err != nil { + str := "failed to create debits bucket" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketUnspent) + if err != nil { + str := "failed to create unspent bucket" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketUnmined) + if err != nil { + str := "failed to create unmined bucket" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketUnminedCredits) + if err != nil { + str := "failed to create unmined credits bucket" + return storeError(ErrDatabase, str, err) + } + + _, err = ns.CreateBucket(bucketUnminedInputs) + if err != nil { + str := "failed to create unmined inputs bucket" + return storeError(ErrDatabase, str, err) + } + + return nil + }) + if err != nil { + const desc = "failed to create new store" + if serr, ok := err.(Error); ok { + serr.Desc = desc + ": " + serr.Desc + return serr + } + return storeError(ErrDatabase, desc, err) + } + + return nil +} + +func scopedUpdate(ns walletdb.Namespace, f func(walletdb.Bucket) error) error { + tx, err := ns.Begin(true) + if err != nil { + str := "cannot begin update" + return storeError(ErrDatabase, str, err) + } + err = f(tx.RootBucket()) + if err != nil { + rbErr := tx.Rollback() + if rbErr != nil { + const desc = "rollback failed" + serr, ok := err.(Error) + if !ok { + // This really shouldn't happen. + return storeError(ErrDatabase, desc, rbErr) + } + serr.Desc = desc + ": " + serr.Desc + return serr + } + return err + } + err = tx.Commit() + if err != nil { + str := "commit failed" + return storeError(ErrDatabase, str, err) + } + return nil +} + +func scopedView(ns walletdb.Namespace, f func(walletdb.Bucket) error) error { + tx, err := ns.Begin(false) + if err != nil { + str := "cannot begin view" + return storeError(ErrDatabase, str, err) + } + err = f(tx.RootBucket()) + rbErr := tx.Rollback() + if err != nil { + return err + } + if rbErr != nil { + str := "cannot close view" + return storeError(ErrDatabase, str, rbErr) + } + return nil +} diff --git a/wtxmgr/doc.go b/wtxmgr/doc.go new file mode 100644 index 0000000..9e90138 --- /dev/null +++ b/wtxmgr/doc.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2013-2015 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +// Package wtxmgr provides an implementation of a transaction database handling +// spend tracking for a bitcoin wallet. Its primary purpose is to save +// transactions with outputs spendable with wallet keys and transactions that +// are signed by wallet keys in memory, handle spend tracking for unspent +// outputs and newly-inserted transactions, and report the spendable balance +// from each unspent transaction output. It uses walletdb as the backend for +// storing the serialized transaction objects in buckets. +// +// Transaction outputs which are spendable by wallet keys are called credits +// (because they credit to a wallet's total spendable balance). Transaction +// inputs which spend previously-inserted credits are called debits (because +// they debit from the wallet's spendable balance). +// +// Spend tracking is mostly automatic. When a new transaction is inserted, if +// it spends from any unspent credits, they are automatically marked spent by +// the new transaction, and each input which spent a credit is marked as a +// debit. However, transaction outputs of inserted transactions must manually +// marked as credits, as this package has no knowledge of wallet keys or +// addresses, and therefore cannot determine which outputs may be spent. +// +// Details regarding individual transactions and their credits and debits may be +// queried either by just a transaction hash, or by hash and block. When +// querying for just a transaction hash, the most recent transaction with a +// matching hash will be queried. However, because transaction hashes may +// collide with other transaction hashes, methods to query for specific +// transactions in the chain (or unmined) are provided as well. +package wtxmgr diff --git a/wtxmgr/error.go b/wtxmgr/error.go new file mode 100644 index 0000000..5c6ed12 --- /dev/null +++ b/wtxmgr/error.go @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2015 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package wtxmgr + +import "fmt" + +// ErrorCode identifies a category of error. +type ErrorCode uint8 + +// These constants are used to identify a specific Error. +const ( + // ErrDatabase indicates an error with the underlying database. When + // this error code is set, the Err field of the Error will be + // set to the underlying error returned from the database. + ErrDatabase ErrorCode = iota + + // ErrData describes an error where data stored in the transaction + // database is incorrect. This may be due to missing values, values of + // wrong sizes, or data from different buckets that is inconsistent with + // itself. Recovering from an ErrData requires rebuilding all + // transaction history or manual database surgery. If the failure was + // not due to data corruption, this error category indicates a + // programming error in this package. + ErrData + + // ErrInput describes an error where the variables passed into this + // function by the caller are obviously incorrect. Examples include + // passing transactions which do not serialize, or attempting to insert + // a credit at an index for which no transaction output exists. + ErrInput + + // ErrAlreadyExists describes an error where creating the store cannot + // continue because a store already exists in the namespace. + ErrAlreadyExists + + // ErrNoExists describes an error where the store cannot be opened due to + // it not already existing in the namespace. This error should be + // handled by creating a new store. + ErrNoExists + + // ErrUnknownVersion describes an error where the store already exists + // but the database version is newer than latest version known to this + // software. This likely indicates an outdated binary. + ErrUnknownVersion +) + +var errStrs = [...]string{ + ErrDatabase: "ErrDatabase", + ErrData: "ErrData", + ErrInput: "ErrInput", + ErrAlreadyExists: "ErrAlreadyExists", + ErrNoExists: "ErrNoExists", + ErrUnknownVersion: "ErrUnknownVersion", +} + +// String returns the ErrorCode as a human-readable name. +func (e ErrorCode) String() string { + if e < ErrorCode(len(errStrs)) { + return errStrs[e] + } + return fmt.Sprintf("ErrorCode(%d)", e) +} + +// Error provides a single type for errors that can happen during Store +// operation. +type Error struct { + Code ErrorCode // Describes the kind of error + Desc string // Human readable description of the issue + Err error // Underlying error, optional +} + +// Error satisfies the error interface and prints human-readable errors. +func (e Error) Error() string { + if e.Err != nil { + return e.Desc + ": " + e.Err.Error() + } + return e.Desc +} + +func storeError(c ErrorCode, desc string, err error) Error { + return Error{Code: c, Desc: desc, Err: err} +} + +// IsNoExists returns whether an error is a Error with the ErrNoExists error +// code. +func IsNoExists(err error) bool { + serr, ok := err.(Error) + return ok && serr.Code == ErrNoExists +} diff --git a/wtxmgr/example_test.go b/wtxmgr/example_test.go new file mode 100644 index 0000000..42edb88 --- /dev/null +++ b/wtxmgr/example_test.go @@ -0,0 +1,233 @@ +// Copyright (c) 2015 Conformal Systems LLC +// +// Permission to use, copy, modify, and distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +package wtxmgr_test + +import ( + "fmt" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcwallet/wtxmgr" +) + +var ( + // Spends: bogus + // Outputs: 10 BTC + exampleTxRecordA *wtxmgr.TxRecord + + // Spends: A:0 + // Outputs: 5 BTC, 5 BTC + exampleTxRecordB *wtxmgr.TxRecord +) + +func init() { + tx := spendOutput(&wire.ShaHash{}, 0, 10e8) + rec, err := wtxmgr.NewTxRecordFromMsgTx(tx, timeNow()) + if err != nil { + panic(err) + } + exampleTxRecordA = rec + + tx = spendOutput(&exampleTxRecordA.Hash, 0, 5e8, 5e8) + rec, err = wtxmgr.NewTxRecordFromMsgTx(tx, timeNow()) + if err != nil { + panic(err) + } + exampleTxRecordB = rec +} + +var exampleBlock100 = makeBlockMeta(100) + +// This example demonstrates reporting the Store balance given an unmined and +// mined transaction given 0, 1, and 6 block confirmations. +func ExampleStore_Balance() { + s, teardown, err := testStore() + defer teardown() + if err != nil { + fmt.Println(err) + return + } + + // Prints balances for 0 block confirmations, 1 confirmation, and 6 + // confirmations. + printBalances := func(syncHeight int32) { + zeroConfBal, err := s.Balance(0, syncHeight) + if err != nil { + fmt.Println(err) + return + } + oneConfBal, err := s.Balance(1, syncHeight) + if err != nil { + fmt.Println(err) + return + } + sixConfBal, err := s.Balance(6, syncHeight) + if err != nil { + fmt.Println(err) + return + } + fmt.Printf("%v, %v, %v\n", zeroConfBal, oneConfBal, sixConfBal) + } + + // Insert a transaction which outputs 10 BTC unmined and mark the output + // as a credit. + err = s.InsertTx(exampleTxRecordA, nil) + if err != nil { + fmt.Println(err) + return + } + err = s.AddCredit(exampleTxRecordA, nil, 0, false) + if err != nil { + fmt.Println(err) + return + } + printBalances(100) + + // Mine the transaction in block 100 and print balances again with a + // sync height of 100 and 105 blocks. + err = s.InsertTx(exampleTxRecordA, &exampleBlock100) + if err != nil { + fmt.Println(err) + return + } + printBalances(100) + printBalances(105) + + // Output: + // 10 BTC, 0 BTC, 0 BTC + // 10 BTC, 10 BTC, 0 BTC + // 10 BTC, 10 BTC, 10 BTC +} + +func ExampleStore_Rollback() { + s, teardown, err := testStore() + defer teardown() + if err != nil { + fmt.Println(err) + return + } + + // Insert a transaction which outputs 10 BTC in a block at height 100. + err = s.InsertTx(exampleTxRecordA, &exampleBlock100) + if err != nil { + fmt.Println(err) + return + } + + // Rollback everything from block 100 onwards. + err = s.Rollback(100) + if err != nil { + fmt.Println(err) + return + } + + // Assert that the transaction is now unmined. + details, err := s.TxDetails(&exampleTxRecordA.Hash) + if err != nil { + fmt.Println(err) + return + } + if details == nil { + fmt.Println("No details found") + return + } + fmt.Println(details.Block.Height) + + // Output: + // -1 +} + +func Example_basicUsage() { + // Open the database. + db, dbTeardown, err := testDB() + defer dbTeardown() + if err != nil { + fmt.Println(err) + return + } + + // Create or open a db namespace for the transaction store. + ns, err := db.Namespace([]byte("txstore")) + if err != nil { + fmt.Println(err) + return + } + + // Create (or open) the transaction store in the provided namespace. + s, err := wtxmgr.Create(ns) + if err != nil { + fmt.Println(err) + return + } + + // Insert an unmined transaction that outputs 10 BTC to a wallet address + // at output 0. + err = s.InsertTx(exampleTxRecordA, nil) + if err != nil { + fmt.Println(err) + return + } + err = s.AddCredit(exampleTxRecordA, nil, 0, false) + if err != nil { + fmt.Println(err) + return + } + + // Insert a second transaction which spends the output, and creates two + // outputs. Mark the second one (5 BTC) as wallet change. + err = s.InsertTx(exampleTxRecordB, nil) + if err != nil { + fmt.Println(err) + return + } + err = s.AddCredit(exampleTxRecordB, nil, 1, true) + if err != nil { + fmt.Println(err) + return + } + + // Mine each transaction in a block at height 100. + err = s.InsertTx(exampleTxRecordA, &exampleBlock100) + if err != nil { + fmt.Println(err) + return + } + err = s.InsertTx(exampleTxRecordB, &exampleBlock100) + if err != nil { + fmt.Println(err) + return + } + + // Print the one confirmation balance. + bal, err := s.Balance(1, 100) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(bal) + + // Fetch unspent outputs. + utxos, err := s.UnspentOutputs() + if err != nil { + fmt.Println(err) + } + expectedOutPoint := wire.OutPoint{Hash: exampleTxRecordB.Hash, Index: 1} + for _, utxo := range utxos { + fmt.Println(utxo.OutPoint == expectedOutPoint) + } + + // Output: + // 5 BTC + // true +} diff --git a/wtxmgr/log.go b/wtxmgr/log.go new file mode 100644 index 0000000..9d9b4c3 --- /dev/null +++ b/wtxmgr/log.go @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2013-2015 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package wtxmgr + +import "github.com/btcsuite/btclog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = btclog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = btclog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using btclog. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/wtxmgr/query.go b/wtxmgr/query.go new file mode 100644 index 0000000..ba1e03d --- /dev/null +++ b/wtxmgr/query.go @@ -0,0 +1,474 @@ +/* + * Copyright (c) 2015 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package wtxmgr + +import ( + "fmt" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwallet/walletdb" +) + +// CreditRecord contains metadata regarding a transaction credit for a known +// transaction. Further details may be looked up by indexing a wire.MsgTx.TxOut +// with the Index field. +type CreditRecord struct { + Index uint32 + Amount btcutil.Amount + Spent bool + Change bool +} + +// DebitRecord contains metadata regarding a transaction debit for a known +// transaction. Further details may be looked up by indexing a wire.MsgTx.TxIn +// with the Index field. +type DebitRecord struct { + Amount btcutil.Amount + Index uint32 +} + +// TxDetails is intended to provide callers with access to rich details +// regarding a relevant transaction and which inputs and outputs are credit or +// debits. +type TxDetails struct { + TxRecord + Block BlockMeta + Credits []CreditRecord + Debits []DebitRecord +} + +// minedTxDetails fetches the TxDetails for the mined transaction with hash +// txHash and the passed tx record key and value. +func (s *Store) minedTxDetails(ns walletdb.Bucket, txHash *wire.ShaHash, recKey, recVal []byte) (*TxDetails, error) { + var details TxDetails + + // Parse transaction record k/v, lookup the full block record for the + // block time, and read all matching credits, debits. + err := readRawTxRecord(txHash, recVal, &details.TxRecord) + if err != nil { + return nil, err + } + err = readRawTxRecordBlock(recKey, &details.Block.Block) + if err != nil { + return nil, err + } + details.Block.Time, err = fetchBlockTime(ns, details.Block.Height) + if err != nil { + return nil, err + } + + credIter := makeCreditIterator(ns, recKey) + for credIter.next() { + if int(credIter.elem.Index) >= len(details.MsgTx.TxOut) { + str := "saved credit index exceeds number of outputs" + return nil, storeError(ErrData, str, nil) + } + + // The credit iterator does not record whether this credit was + // spent by an unmined transaction, so check that here. + if !credIter.elem.Spent { + k := canonicalOutPoint(txHash, credIter.elem.Index) + spent := existsRawUnminedInput(ns, k) != nil + credIter.elem.Spent = spent + } + details.Credits = append(details.Credits, credIter.elem) + } + if credIter.err != nil { + return nil, credIter.err + } + + debIter := makeDebitIterator(ns, recKey) + for debIter.next() { + if int(debIter.elem.Index) >= len(details.MsgTx.TxIn) { + str := "saved debit index exceeds number of inputs" + return nil, storeError(ErrData, str, nil) + } + + details.Debits = append(details.Debits, debIter.elem) + } + return &details, debIter.err +} + +// unminedTxDetails fetches the TxDetails for the unmined transaction with the +// hash txHash and the passed unmined record value. +func (s *Store) unminedTxDetails(ns walletdb.Bucket, txHash *wire.ShaHash, v []byte) (*TxDetails, error) { + details := TxDetails{ + Block: BlockMeta{Block: Block{Height: -1}}, + } + err := readRawTxRecord(txHash, v, &details.TxRecord) + if err != nil { + return nil, err + } + + it := makeUnminedCreditIterator(ns, txHash) + for it.next() { + if int(it.elem.Index) >= len(details.MsgTx.TxOut) { + str := "saved credit index exceeds number of outputs" + return nil, storeError(ErrData, str, nil) + } + + // Set the Spent field since this is not done by the iterator. + it.elem.Spent = existsRawUnminedInput(ns, it.ck) != nil + details.Credits = append(details.Credits, it.elem) + } + if it.err != nil { + return nil, it.err + } + + // Debit records are not saved for unmined transactions. Instead, they + // must be looked up for each transaction input manually. There are two + // kinds of previous credits that may be debited by an unmined + // transaction: mined unspent outputs (which remain marked unspent even + // when spent by an unmined transaction), and credits from other unmined + // transactions. Both situations must be considered. + for i, output := range details.MsgTx.TxIn { + opKey := canonicalOutPoint(&output.PreviousOutPoint.Hash, + output.PreviousOutPoint.Index) + credKey := existsRawUnspent(ns, opKey) + if credKey != nil { + v := existsRawCredit(ns, credKey) + amount, err := fetchRawCreditAmount(v) + if err != nil { + return nil, err + } + + details.Debits = append(details.Debits, DebitRecord{ + Amount: amount, + Index: uint32(i), + }) + continue + } + + v := existsRawUnminedCredit(ns, opKey) + if v == nil { + continue + } + + amount, err := fetchRawCreditAmount(v) + if err != nil { + return nil, err + } + details.Debits = append(details.Debits, DebitRecord{ + Amount: amount, + Index: uint32(i), + }) + } + + return &details, nil +} + +// TxDetails looks up all recorded details regarding a transaction with some +// hash. In case of a hash collision, the most recent transaction with a +// matching hash is returned. +// +// Not finding a transaction with this hash is not an error. In this case, +// a nil TxDetails is returned. +func (s *Store) TxDetails(txHash *wire.ShaHash) (*TxDetails, error) { + var details *TxDetails + err := scopedView(s.namespace, func(ns walletdb.Bucket) error { + var err error + + // First, check whether there exists an unmined transaction with this + // hash. Use it if found. + v := existsRawUnmined(ns, txHash[:]) + if v != nil { + details, err = s.unminedTxDetails(ns, txHash, v) + return err + } + + // Otherwise, if there exists a mined transaction with this matching + // hash, skip over to the newest and begin fetching all details. + k, v := latestTxRecord(ns, txHash) + if v == nil { + // not found + return nil + } + details, err = s.minedTxDetails(ns, txHash, k, v) + return err + }) + return details, err +} + +// UniqueTxDetails looks up all recorded details for a transaction recorded +// mined in some particular block, or an unmined transaction if block is nil. +// +// Not finding a transaction with this hash from this block is not an error. In +// this case, a nil TxDetails is returned. +func (s *Store) UniqueTxDetails(txHash *wire.ShaHash, block *Block) (*TxDetails, error) { + var details *TxDetails + err := scopedView(s.namespace, func(ns walletdb.Bucket) error { + var err error + if block == nil { + v := existsRawUnmined(ns, txHash[:]) + if v == nil { + return nil + } + details, err = s.unminedTxDetails(ns, txHash, v) + return err + } + + k, v := existsTxRecord(ns, txHash, block) + if v == nil { + return nil + } + details, err = s.minedTxDetails(ns, txHash, k, v) + return err + }) + return details, err +} + +// rangeUnminedTransactions executes the function f with TxDetails for every +// unmined transaction. f is not executed if no unmined transactions exist. +// Error returns from f (if any) are propigated to the caller. Returns true +// (signaling breaking out of a RangeTransactions) iff f executes and returns +// true. +func (s *Store) rangeUnminedTransactions(ns walletdb.Bucket, f func([]TxDetails) (bool, error)) (bool, error) { + var details []TxDetails + err := ns.Bucket(bucketUnmined).ForEach(func(k, v []byte) error { + if len(k) < 32 { + str := fmt.Sprintf("%s: short key (expected %d "+ + "bytes, read %d)", bucketUnmined, 32, len(k)) + return storeError(ErrData, str, nil) + } + + var txHash wire.ShaHash + copy(txHash[:], k) + detail, err := s.unminedTxDetails(ns, &txHash, v) + if err != nil { + return err + } + + // Because the key was created while foreach-ing over the + // bucket, it should be impossible for unminedTxDetails to ever + // successfully return a nil details struct. + details = append(details, *detail) + return nil + }) + if err == nil && len(details) > 0 { + return f(details) + } + return false, err +} + +// rangeBlockTransactions executes the function f with TxDetails for every block +// between heights begin and end (reverse order when end > begin) until f +// returns true, or the transactions from block is processed. Returns true iff +// f executes and returns true. +func (s *Store) rangeBlockTransactions(ns walletdb.Bucket, begin, end int32, f func([]TxDetails) (bool, error)) (bool, error) { + // Mempool height is considered a high bound. + if begin < 0 { + begin = int32(^uint32(0) >> 1) + } + if end < 0 { + end = int32(^uint32(0) >> 1) + } + + var blockIter blockIterator + var advance func(*blockIterator) bool + if begin < end { + // Iterate in forwards order + blockIter = makeBlockIterator(ns, begin) + advance = func(it *blockIterator) bool { + if !it.next() { + return false + } + return it.elem.Height <= end + } + } else { + // Iterate in backwards order, from begin -> end. + blockIter = makeBlockIterator(ns, begin) + advance = func(it *blockIterator) bool { + if !it.prev() { + return false + } + return end <= it.elem.Height + } + } + + var details []TxDetails + for advance(&blockIter) { + block := &blockIter.elem + + if cap(details) < len(block.transactions) { + details = make([]TxDetails, 0, len(block.transactions)) + } else { + details = details[:0] + } + + for _, txHash := range block.transactions { + k := keyTxRecord(&txHash, &block.Block) + v := existsRawTxRecord(ns, k) + if v == nil { + str := fmt.Sprintf("missing transaction %v for "+ + "block %v", txHash, block.Height) + return false, storeError(ErrData, str, nil) + } + detail := TxDetails{ + Block: BlockMeta{ + Block: block.Block, + Time: block.Time, + }, + } + err := readRawTxRecord(&txHash, v, &detail.TxRecord) + if err != nil { + return false, err + } + + credIter := makeCreditIterator(ns, k) + for credIter.next() { + if int(credIter.elem.Index) >= len(detail.MsgTx.TxOut) { + str := "saved credit index exceeds number of outputs" + return false, storeError(ErrData, str, nil) + } + + // The credit iterator does not record whether + // this credit was spent by an unmined + // transaction, so check that here. + if !credIter.elem.Spent { + k := canonicalOutPoint(&txHash, credIter.elem.Index) + spent := existsRawUnminedInput(ns, k) != nil + credIter.elem.Spent = spent + } + detail.Credits = append(detail.Credits, credIter.elem) + } + if credIter.err != nil { + return false, credIter.err + } + + debIter := makeDebitIterator(ns, k) + for debIter.next() { + if int(debIter.elem.Index) >= len(detail.MsgTx.TxIn) { + str := "saved debit index exceeds number of inputs" + return false, storeError(ErrData, str, nil) + } + + detail.Debits = append(detail.Debits, debIter.elem) + } + if debIter.err != nil { + return false, debIter.err + } + + details = append(details, detail) + } + + // Every block record must have at least one transaction, so it + // is safe to call f. + brk, err := f(details) + if err != nil || brk { + return brk, err + } + } + return false, blockIter.err +} + +// RangeTransactions runs the function f on all transaction details between +// blocks on the best chain over the height range [begin,end]. The special +// height -1 may be used to also include unmined transactions. If the end +// height comes before the begin height, blocks are iterated in reverse order +// and unmined transactions (if any) are processed first. +// +// The function f may return an error which, if non-nil, is propagated to the +// caller. Additionally, a boolean return value allows exiting the function +// early without reading any additional transactions early when true. +// +// All calls to f are guaranteed to be passed a slice with more than zero +// elements. The slice may be reused for multiple blocks, so it is not safe to +// use it after the loop iteration it was acquired. +func (s *Store) RangeTransactions(begin, end int32, f func([]TxDetails) (bool, error)) error { + return scopedView(s.namespace, func(ns walletdb.Bucket) error { + var addedUnmined bool + if begin < 0 { + brk, err := s.rangeUnminedTransactions(ns, f) + if err != nil || brk { + return err + } + addedUnmined = true + } + + brk, err := s.rangeBlockTransactions(ns, begin, end, f) + if err == nil && !brk && !addedUnmined && end < 0 { + _, err = s.rangeUnminedTransactions(ns, f) + } + return err + }) +} + +// PreviousPkScripts returns a slice of previous output scripts for each credit +// output this transaction record debits from. +func (s *Store) PreviousPkScripts(rec *TxRecord, block *Block) ([][]byte, error) { + var pkScripts [][]byte + err := scopedView(s.namespace, func(ns walletdb.Bucket) error { + if block == nil { + for _, input := range rec.MsgTx.TxIn { + prevOut := &input.PreviousOutPoint + + // Input may spend a previous unmined output, a + // mined output (which would still be marked + // unspent), or neither. + + v := existsRawUnmined(ns, prevOut.Hash[:]) + if v != nil { + // Ensure a credit exists for this + // unmined transaction before including + // the output script. + k := canonicalOutPoint(&prevOut.Hash, prevOut.Index) + if existsRawUnminedCredit(ns, k) == nil { + continue + } + + pkScript, err := fetchRawTxRecordPkScript( + prevOut.Hash[:], v, prevOut.Index) + if err != nil { + return err + } + pkScripts = append(pkScripts, pkScript) + continue + } + + _, credKey := existsUnspent(ns, prevOut) + if credKey != nil { + k := extractRawCreditTxRecordKey(credKey) + v = existsRawTxRecord(ns, k) + pkScript, err := fetchRawTxRecordPkScript(k, v, + prevOut.Index) + if err != nil { + return err + } + pkScripts = append(pkScripts, pkScript) + } + } + return nil + } + + recKey := keyTxRecord(&rec.Hash, block) + it := makeDebitIterator(ns, recKey) + for it.next() { + credKey := extractRawDebitCreditKey(it.cv) + index := extractRawCreditIndex(credKey) + k := extractRawCreditTxRecordKey(credKey) + v := existsRawTxRecord(ns, k) + pkScript, err := fetchRawTxRecordPkScript(k, v, index) + if err != nil { + return err + } + pkScripts = append(pkScripts, pkScript) + } + return it.err + }) + return pkScripts, err +} diff --git a/wtxmgr/query_test.go b/wtxmgr/query_test.go new file mode 100644 index 0000000..f35420d --- /dev/null +++ b/wtxmgr/query_test.go @@ -0,0 +1,744 @@ +// Copyright (c) 2015 Conformal Systems LLC +// +// Permission to use, copy, modify, and distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +package wtxmgr_test + +import ( + "bytes" + "encoding/binary" + "fmt" + "testing" + "time" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + . "github.com/btcsuite/btcwallet/wtxmgr" +) + +type queryState struct { + // slice items are ordered by height, mempool comes last. + blocks [][]TxDetails + txDetails map[wire.ShaHash][]TxDetails +} + +func newQueryState() *queryState { + return &queryState{ + txDetails: make(map[wire.ShaHash][]TxDetails), + } +} + +func (q *queryState) deepCopy() *queryState { + cpy := newQueryState() + for _, blockDetails := range q.blocks { + var cpyDetails []TxDetails + for _, detail := range blockDetails { + cpyDetails = append(cpyDetails, *deepCopyTxDetails(&detail)) + } + cpy.blocks = append(cpy.blocks, cpyDetails) + } + cpy.txDetails = make(map[wire.ShaHash][]TxDetails) + for txHash, details := range q.txDetails { + detailsSlice := make([]TxDetails, len(details)) + for i, detail := range details { + detailsSlice[i] = *deepCopyTxDetails(&detail) + } + cpy.txDetails[txHash] = detailsSlice + } + return cpy +} + +func deepCopyTxDetails(d *TxDetails) *TxDetails { + cpy := *d + cpy.MsgTx = *d.MsgTx.Copy() + if cpy.SerializedTx != nil { + cpy.SerializedTx = make([]byte, len(cpy.SerializedTx)) + copy(cpy.SerializedTx, d.SerializedTx) + } + cpy.Credits = make([]CreditRecord, len(d.Credits)) + copy(cpy.Credits, d.Credits) + cpy.Debits = make([]DebitRecord, len(d.Debits)) + copy(cpy.Debits, d.Debits) + return &cpy +} + +func (q *queryState) compare(t *testing.T, s *Store, changeDesc string) { + defer func() { + if t.Failed() { + t.Fatalf("Store state queries failed after '%s'", changeDesc) + } + }() + + fwdBlocks := q.blocks + revBlocks := make([][]TxDetails, len(q.blocks)) + copy(revBlocks, q.blocks) + for i := 0; i < len(revBlocks)/2; i++ { + revBlocks[i], revBlocks[len(revBlocks)-1-i] = revBlocks[len(revBlocks)-1-i], revBlocks[i] + } + checkBlock := func(blocks [][]TxDetails) func([]TxDetails) (bool, error) { + return func(got []TxDetails) (bool, error) { + if len(fwdBlocks) == 0 { + return false, fmt.Errorf("entered range when no more details expected") + } + exp := blocks[0] + if len(got) != len(exp) { + return false, fmt.Errorf("got len(details)=%d in transaction range, expected %d", len(got), len(exp)) + } + for i := range got { + equalTxDetails(t, &got[i], &exp[i]) + } + if t.Failed() { + return false, fmt.Errorf("Failed comparing range of transaction details") + } + blocks = blocks[1:] + return false, nil + } + } + err := s.RangeTransactions(0, -1, checkBlock(fwdBlocks)) + if err != nil { + t.Fatalf("Failed in RangeTransactions (forwards iteration): %v", err) + } + err = s.RangeTransactions(-1, 0, checkBlock(revBlocks)) + if err != nil { + t.Fatalf("Failed in RangeTransactions (reverse iteration): %v", err) + } + + for txHash, details := range q.txDetails { + for _, detail := range details { + blk := &detail.Block.Block + if blk.Height == -1 { + blk = nil + } + d, err := s.UniqueTxDetails(&txHash, blk) + if err != nil { + t.Fatal(err) + } + if d == nil { + t.Errorf("Found no matching transaction at height %d", detail.Block.Height) + continue + } + equalTxDetails(t, d, &detail) + } + if t.Failed() { + t.Fatalf("Failed querying unique details regarding transaction %v", txHash) + } + + // For the most recent tx with this hash, check that + // TxDetails (not looking up a tx at any particular + // height) matches the last. + detail := &details[len(details)-1] + d, err := s.TxDetails(&txHash) + if err != nil { + t.Fatal(err) + } + equalTxDetails(t, d, detail) + if t.Failed() { + t.Fatalf("Failed querying latest details regarding transaction %v", txHash) + } + } +} + +func equalTxDetails(t *testing.T, got, exp *TxDetails) { + // Need to avoid using reflect.DeepEqual against slices, since it + // returns false for nil vs non-nil zero length slices. + + equalTxs(t, &got.MsgTx, &exp.MsgTx) + if got.Hash != exp.Hash { + t.Errorf("Found mismatched hashes") + t.Errorf("Got: %v", got.Hash) + t.Errorf("Expected: %v", exp.Hash) + } + if got.Received != exp.Received { + t.Errorf("Found mismatched receive time") + t.Errorf("Got: %v", got.Received) + t.Errorf("Expected: %v", exp.Received) + } + if !bytes.Equal(got.SerializedTx, exp.SerializedTx) { + t.Errorf("Found mismatched serialized txs") + t.Errorf("Got: %x", got.SerializedTx) + t.Errorf("Expected: %x", exp.SerializedTx) + } + if got.Block != exp.Block { + t.Errorf("Found mismatched block meta") + t.Errorf("Got: %v", got.Block) + t.Errorf("Expected: %v", exp.Block) + } + if len(got.Credits) != len(exp.Credits) { + t.Errorf("Credit slice lengths differ: Got %d Expected %d", len(got.Credits), len(exp.Credits)) + } else { + for i := range got.Credits { + if got.Credits[i] != exp.Credits[i] { + t.Errorf("Found mismatched Credit[%d]", i) + t.Errorf("Got: %v", got.Credits[i]) + t.Errorf("Expected: %v", exp.Credits[i]) + } + } + } + if len(got.Debits) != len(exp.Debits) { + t.Errorf("Debit slice lengths differ: Got %d Expected %d", len(got.Debits), len(exp.Debits)) + } else { + for i := range got.Debits { + if got.Debits[i] != exp.Debits[i] { + t.Errorf("Found mismatched Debit[%d]", i) + t.Errorf("Got: %v", got.Debits[i]) + t.Errorf("Expected: %v", exp.Debits[i]) + } + } + } +} + +func equalTxs(t *testing.T, got, exp *wire.MsgTx) { + var bufGot, bufExp bytes.Buffer + err := got.Serialize(&bufGot) + if err != nil { + t.Fatal(err) + } + err = exp.Serialize(&bufExp) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(bufGot.Bytes(), bufExp.Bytes()) { + t.Errorf("Found unexpected wire.MsgTx:") + t.Errorf("Got: %v", got) + t.Errorf("Expected: %v", exp) + } +} + +// Returns time.Now() with seconds resolution, this is what Store saves. +func timeNow() time.Time { + return time.Unix(time.Now().Unix(), 0) +} + +// Returns a copy of a TxRecord without the serialized tx. +func stripSerializedTx(rec *TxRecord) *TxRecord { + ret := *rec + ret.SerializedTx = nil + return &ret +} + +func makeBlockMeta(height int32) BlockMeta { + if height == -1 { + return BlockMeta{Block: Block{Height: -1}} + } + + b := BlockMeta{ + Block: Block{Height: height}, + Time: timeNow(), + } + // Give it a fake block hash created from the height and time. + binary.LittleEndian.PutUint32(b.Hash[0:4], uint32(height)) + binary.LittleEndian.PutUint64(b.Hash[4:12], uint64(b.Time.Unix())) + return b +} + +func TestStoreQueries(t *testing.T) { + t.Parallel() + + type queryTest struct { + desc string + updates func() // Unwinds from t.Fatal if the update errors. + state *queryState + } + var tests []queryTest + + // Create the store and test initial state. + s, teardown, err := testStore() + defer teardown() + if err != nil { + t.Fatal(err) + } + lastState := newQueryState() + tests = append(tests, queryTest{ + desc: "initial store", + updates: func() {}, + state: lastState, + }) + + // simplify error handling + insertTx := func(rec *TxRecord, block *BlockMeta) { + err := s.InsertTx(rec, block) + if err != nil { + t.Fatal(err) + } + } + addCredit := func(s *Store, rec *TxRecord, block *BlockMeta, index uint32, change bool) { + err := s.AddCredit(rec, block, index, change) + if err != nil { + t.Fatal(err) + } + } + newTxRecordFromMsgTx := func(tx *wire.MsgTx, received time.Time) *TxRecord { + rec, err := NewTxRecordFromMsgTx(tx, received) + if err != nil { + t.Fatal(err) + } + return rec + } + rollback := func(height int32) { + err := s.Rollback(height) + if err != nil { + t.Fatal(err) + } + } + + // Insert an unmined transaction. Mark no credits yet. + txA := spendOutput(&wire.ShaHash{}, 0, 100e8) + recA := newTxRecordFromMsgTx(txA, timeNow()) + newState := lastState.deepCopy() + newState.blocks = [][]TxDetails{ + { + { + TxRecord: *stripSerializedTx(recA), + Block: BlockMeta{Block: Block{Height: -1}}, + }, + }, + } + newState.txDetails[recA.Hash] = []TxDetails{ + newState.blocks[0][0], + } + lastState = newState + tests = append(tests, queryTest{ + desc: "insert tx A unmined", + updates: func() { insertTx(recA, nil) }, + state: newState, + }) + + // Add txA:0 as a change credit. + newState = lastState.deepCopy() + newState.blocks[0][0].Credits = []CreditRecord{ + { + Index: 0, + Amount: btcutil.Amount(recA.MsgTx.TxOut[0].Value), + Spent: false, + Change: true, + }, + } + newState.txDetails[recA.Hash][0].Credits = newState.blocks[0][0].Credits + lastState = newState + tests = append(tests, queryTest{ + desc: "mark unconfirmed txA:0 as credit", + updates: func() { addCredit(s, recA, nil, 0, true) }, + state: newState, + }) + + // Insert another unmined transaction which spends txA:0, splitting the + // amount into outputs of 40 and 60 BTC. + txB := spendOutput(&recA.Hash, 0, 40e8, 60e8) + recB := newTxRecordFromMsgTx(txB, timeNow()) + newState = lastState.deepCopy() + newState.blocks[0][0].Credits[0].Spent = true + newState.blocks[0] = append(newState.blocks[0], TxDetails{ + TxRecord: *stripSerializedTx(recB), + Block: BlockMeta{Block: Block{Height: -1}}, + Debits: []DebitRecord{ + { + Amount: btcutil.Amount(recA.MsgTx.TxOut[0].Value), + Index: 0, // recB.MsgTx.TxIn index + }, + }, + }) + newState.txDetails[recA.Hash][0].Credits[0].Spent = true + newState.txDetails[recB.Hash] = []TxDetails{newState.blocks[0][1]} + lastState = newState + tests = append(tests, queryTest{ + desc: "insert tx B unmined", + updates: func() { insertTx(recB, nil) }, + state: newState, + }) + newState = lastState.deepCopy() + newState.blocks[0][1].Credits = []CreditRecord{ + { + Index: 0, + Amount: btcutil.Amount(recB.MsgTx.TxOut[0].Value), + Spent: false, + Change: false, + }, + } + newState.txDetails[recB.Hash][0].Credits = newState.blocks[0][1].Credits + lastState = newState + tests = append(tests, queryTest{ + desc: "mark txB:0 as non-change credit", + updates: func() { addCredit(s, recB, nil, 0, false) }, + state: newState, + }) + + // Mine tx A at block 100. Leave tx B unmined. + b100 := makeBlockMeta(100) + newState = lastState.deepCopy() + newState.blocks[0] = newState.blocks[0][:1] + newState.blocks[0][0].Block = b100 + newState.blocks = append(newState.blocks, lastState.blocks[0][1:]) + newState.txDetails[recA.Hash][0].Block = b100 + lastState = newState + tests = append(tests, queryTest{ + desc: "mine tx A", + updates: func() { insertTx(recA, &b100) }, + state: newState, + }) + + // Mine tx B at block 101. + b101 := makeBlockMeta(101) + newState = lastState.deepCopy() + newState.blocks[1][0].Block = b101 + newState.txDetails[recB.Hash][0].Block = b101 + lastState = newState + tests = append(tests, queryTest{ + desc: "mine tx B", + updates: func() { insertTx(recB, &b101) }, + state: newState, + }) + + for _, tst := range tests { + tst.updates() + tst.state.compare(t, s, tst.desc) + } + + // Run some additional query tests with the current store's state: + // - Verify that querying for a transaction not in the store returns + // nil without failure. + // - Verify that querying for a unique transaction at the wrong block + // returns nil without failure. + // - Verify that breaking early on RangeTransactions stops further + // iteration. + + missingTx := spendOutput(&recB.Hash, 0, 40e8) + missingRec := newTxRecordFromMsgTx(missingTx, timeNow()) + missingBlock := makeBlockMeta(102) + missingDetails, err := s.TxDetails(&missingRec.Hash) + if err != nil { + t.Fatal(err) + } + if missingDetails != nil { + t.Errorf("Expected no details, found details for tx %v", missingDetails.Hash) + } + missingUniqueTests := []struct { + hash *wire.ShaHash + block *Block + }{ + {&missingRec.Hash, &b100.Block}, + {&missingRec.Hash, &missingBlock.Block}, + {&missingRec.Hash, nil}, + {&recB.Hash, &b100.Block}, + {&recB.Hash, &missingBlock.Block}, + {&recB.Hash, nil}, + } + for _, tst := range missingUniqueTests { + missingDetails, err = s.UniqueTxDetails(tst.hash, tst.block) + if err != nil { + t.Fatal(err) + } + if missingDetails != nil { + t.Errorf("Expected no details, found details for tx %v", missingDetails.Hash) + } + } + + iterations := 0 + err = s.RangeTransactions(0, -1, func([]TxDetails) (bool, error) { + iterations++ + return true, nil + }) + if iterations != 1 { + t.Errorf("RangeTransactions (forwards) ran func %d times", iterations) + } + iterations = 0 + err = s.RangeTransactions(-1, 0, func([]TxDetails) (bool, error) { + iterations++ + return true, nil + }) + if iterations != 1 { + t.Errorf("RangeTransactions (reverse) ran func %d times", iterations) + } + // Make sure it also breaks early after one iteration through unmined transactions. + rollback(b101.Height) + iterations = 0 + err = s.RangeTransactions(-1, 0, func([]TxDetails) (bool, error) { + iterations++ + return true, nil + }) + if iterations != 1 { + t.Errorf("RangeTransactions (reverse) ran func %d times", iterations) + } + + // None of the above tests have tested RangeTransactions with multiple + // txs per block, so do that now. Start by moving tx B to block 100 + // (same block as tx A), and then rollback from block 100 onwards so + // both are unmined. + newState = lastState.deepCopy() + newState.blocks[0] = append(newState.blocks[0], newState.blocks[1]...) + newState.blocks[0][1].Block = b100 + newState.blocks = newState.blocks[:1] + newState.txDetails[recB.Hash][0].Block = b100 + lastState = newState + tests = append(tests[:0:0], queryTest{ + desc: "move tx B to block 100", + updates: func() { insertTx(recB, &b100) }, + state: newState, + }) + newState = lastState.deepCopy() + newState.blocks[0][0].Block = makeBlockMeta(-1) + newState.blocks[0][1].Block = makeBlockMeta(-1) + newState.txDetails[recA.Hash][0].Block = makeBlockMeta(-1) + newState.txDetails[recB.Hash][0].Block = makeBlockMeta(-1) + lastState = newState + tests = append(tests, queryTest{ + desc: "rollback block 100", + updates: func() { rollback(b100.Height) }, + state: newState, + }) + + // None of the above tests have tested transactions with colliding + // hashes, so mine tx A in block 100, and then insert tx A again + // unmined. Also mine tx A in block 101 (this moves it from unmined). + // This is a valid test because the store does not perform signature + // validation or keep a full utxo set, and duplicated transaction hashes + // from different blocks are allowed so long as all previous outputs are + // spent. + newState = lastState.deepCopy() + newState.blocks = append(newState.blocks, newState.blocks[0][1:]) + newState.blocks[0] = newState.blocks[0][:1:1] + newState.blocks[0][0].Block = b100 + newState.blocks[1] = []TxDetails{ + { + TxRecord: *stripSerializedTx(recA), + Block: makeBlockMeta(-1), + }, + newState.blocks[1][0], + } + newState.txDetails[recA.Hash][0].Block = b100 + newState.txDetails[recA.Hash] = append(newState.txDetails[recA.Hash], newState.blocks[1][0]) + lastState = newState + tests = append(tests, queryTest{ + desc: "insert duplicate tx A", + updates: func() { insertTx(recA, &b100); insertTx(recA, nil) }, + state: newState, + }) + newState = lastState.deepCopy() + newState.blocks = [][]TxDetails{ + newState.blocks[0], + []TxDetails{newState.blocks[1][0]}, + []TxDetails{newState.blocks[1][1]}, + } + newState.blocks[1][0].Block = b101 + newState.txDetails[recA.Hash][1].Block = b101 + lastState = newState + tests = append(tests, queryTest{ + desc: "mine duplicate tx A", + updates: func() { insertTx(recA, &b101) }, + state: newState, + }) + + for _, tst := range tests { + tst.updates() + tst.state.compare(t, s, tst.desc) + } +} + +func TestPreviousPkScripts(t *testing.T) { + t.Parallel() + + s, teardown, err := testStore() + defer teardown() + if err != nil { + t.Fatal(err) + } + + // Invalid scripts but sufficient for testing. + var ( + scriptA0 = []byte("tx A output 0") + scriptA1 = []byte("tx A output 1") + scriptB0 = []byte("tx B output 0") + scriptB1 = []byte("tx B output 1") + scriptC0 = []byte("tx C output 0") + scriptC1 = []byte("tx C output 1") + ) + + // Create a transaction spending two prevous outputs and generating two + // new outputs the passed pkScipts. Spends outputs 0 and 1 from prevHash. + buildTx := func(prevHash *wire.ShaHash, script0, script1 []byte) *wire.MsgTx { + return &wire.MsgTx{ + TxIn: []*wire.TxIn{ + &wire.TxIn{PreviousOutPoint: wire.OutPoint{*prevHash, 0}}, + &wire.TxIn{PreviousOutPoint: wire.OutPoint{*prevHash, 1}}, + }, + TxOut: []*wire.TxOut{ + &wire.TxOut{Value: 1e8, PkScript: script0}, + &wire.TxOut{Value: 1e8, PkScript: script1}, + }, + } + } + + newTxRecordFromMsgTx := func(tx *wire.MsgTx) *TxRecord { + rec, err := NewTxRecordFromMsgTx(tx, timeNow()) + if err != nil { + t.Fatal(err) + } + return rec + } + + // Create transactions with the fake output scripts. + var ( + txA = buildTx(&wire.ShaHash{}, scriptA0, scriptA1) + recA = newTxRecordFromMsgTx(txA) + txB = buildTx(&recA.Hash, scriptB0, scriptB1) + recB = newTxRecordFromMsgTx(txB) + txC = buildTx(&recB.Hash, scriptC0, scriptC1) + recC = newTxRecordFromMsgTx(txC) + txD = buildTx(&recC.Hash, nil, nil) + recD = newTxRecordFromMsgTx(txD) + ) + + insertTx := func(rec *TxRecord, block *BlockMeta) { + err := s.InsertTx(rec, block) + if err != nil { + t.Fatal(err) + } + } + addCredit := func(rec *TxRecord, block *BlockMeta, index uint32) { + err := s.AddCredit(rec, block, index, false) + if err != nil { + t.Fatal(err) + } + } + + type scriptTest struct { + rec *TxRecord + block *Block + scripts [][]byte + } + runTest := func(tst *scriptTest) { + scripts, err := s.PreviousPkScripts(tst.rec, tst.block) + if err != nil { + t.Fatal(err) + } + height := int32(-1) + if tst.block != nil { + height = tst.block.Height + } + if len(scripts) != len(tst.scripts) { + t.Errorf("Transaction %v height %d: got len(scripts)=%d, expected %d", + tst.rec.Hash, height, len(scripts), len(tst.scripts)) + return + } + for i := range scripts { + if !bytes.Equal(scripts[i], tst.scripts[i]) { + // Format scripts with %s since they are (should be) ascii. + t.Errorf("Transaction %v height %d script %d: got '%s' expected '%s'", + tst.rec.Hash, height, i, scripts[i], tst.scripts[i]) + } + } + } + + // Insert transactions A-C unmined, but mark no credits yet. Until + // these are marked as credits, PreviousPkScripts should not return + // them. + insertTx(recA, nil) + insertTx(recB, nil) + insertTx(recC, nil) + + b100 := makeBlockMeta(100) + b101 := makeBlockMeta(101) + + tests := []scriptTest{ + {recA, nil, nil}, + {recA, &b100.Block, nil}, + {recB, nil, nil}, + {recB, &b100.Block, nil}, + {recC, nil, nil}, + {recC, &b100.Block, nil}, + } + for _, tst := range tests { + runTest(&tst) + } + if t.Failed() { + t.Fatal("Failed after unmined tx inserts") + } + + // Mark credits. Tx C output 1 not marked as a credit: tx D will spend + // both later but when C is mined, output 1's script should not be + // returned. + addCredit(recA, nil, 0) + addCredit(recA, nil, 1) + addCredit(recB, nil, 0) + addCredit(recB, nil, 1) + addCredit(recC, nil, 0) + tests = []scriptTest{ + {recA, nil, nil}, + {recA, &b100.Block, nil}, + {recB, nil, [][]byte{scriptA0, scriptA1}}, + {recB, &b100.Block, nil}, + {recC, nil, [][]byte{scriptB0, scriptB1}}, + {recC, &b100.Block, nil}, + } + for _, tst := range tests { + runTest(&tst) + } + if t.Failed() { + t.Fatal("Failed after marking unmined credits") + } + + // Mine tx A in block 100. Test results should be identical. + insertTx(recA, &b100) + for _, tst := range tests { + runTest(&tst) + } + if t.Failed() { + t.Fatal("Failed after mining tx A") + } + + // Mine tx B in block 101. + insertTx(recB, &b101) + tests = []scriptTest{ + {recA, nil, nil}, + {recA, &b100.Block, nil}, + {recB, nil, nil}, + {recB, &b101.Block, [][]byte{scriptA0, scriptA1}}, + {recC, nil, [][]byte{scriptB0, scriptB1}}, + {recC, &b101.Block, nil}, + } + for _, tst := range tests { + runTest(&tst) + } + if t.Failed() { + t.Fatal("Failed after mining tx B") + } + + // Mine tx C in block 101 (same block as tx B) to test debits from the + // same block. + insertTx(recC, &b101) + tests = []scriptTest{ + {recA, nil, nil}, + {recA, &b100.Block, nil}, + {recB, nil, nil}, + {recB, &b101.Block, [][]byte{scriptA0, scriptA1}}, + {recC, nil, nil}, + {recC, &b101.Block, [][]byte{scriptB0, scriptB1}}, + } + for _, tst := range tests { + runTest(&tst) + } + if t.Failed() { + t.Fatal("Failed after mining tx C") + } + + // Insert tx D, which spends C:0 and C:1. However, only C:0 is marked + // as a credit, and only that output script should be returned. + insertTx(recD, nil) + tests = append(tests, scriptTest{recD, nil, [][]byte{scriptC0}}) + tests = append(tests, scriptTest{recD, &b101.Block, nil}) + for _, tst := range tests { + runTest(&tst) + } + if t.Failed() { + t.Fatal("Failed after inserting tx D") + } +} diff --git a/wtxmgr/tx.go b/wtxmgr/tx.go new file mode 100644 index 0000000..7c9ad9c --- /dev/null +++ b/wtxmgr/tx.go @@ -0,0 +1,936 @@ +/* + * Copyright (c) 2013-2015 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package wtxmgr + +import ( + "bytes" + "time" + + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwallet/walletdb" +) + +// Block contains the minimum amount of data to uniquely identify any block on +// either the best or side chain. +type Block struct { + Hash wire.ShaHash + Height int32 +} + +// BlockMeta contains the unique identification for a block and any metadata +// pertaining to the block. At the moment, this additional metadata only +// includes the block time from the block header. +type BlockMeta struct { + Block + Time time.Time +} + +// blockRecord is an in-memory representation of the block record saved in the +// database. +type blockRecord struct { + Block + Time time.Time + transactions []wire.ShaHash +} + +// incidence records the block hash and blockchain height of a mined transaction. +// Since a transaction hash alone is not enough to uniquely identify a mined +// transaction (duplicate transaction hashes are allowed), the incidence is used +// instead. +type incidence struct { + txHash wire.ShaHash + block Block +} + +// indexedIncidence records the transaction incidence and an input or output +// index. +type indexedIncidence struct { + incidence + index uint32 +} + +// debit records the debits a transaction record makes from previous wallet +// transaction credits. +type debit struct { + txHash wire.ShaHash + index uint32 + amount btcutil.Amount + spends indexedIncidence +} + +// credit describes a transaction output which was or is spendable by wallet. +type credit struct { + outPoint wire.OutPoint + block Block + amount btcutil.Amount + change bool + spentBy indexedIncidence // Index == ^uint32(0) if unspent +} + +// TxRecord represents a transaction managed by the Store. +type TxRecord struct { + MsgTx wire.MsgTx + Hash wire.ShaHash + Received time.Time + SerializedTx []byte // Optional: may be nil +} + +// NewTxRecord creates a new transaction record that may be inserted into the +// store. It uses memoization to save the transaction hash and the serialized +// transaction. +func NewTxRecord(serializedTx []byte, received time.Time) (*TxRecord, error) { + rec := &TxRecord{ + Received: received, + SerializedTx: serializedTx, + } + err := rec.MsgTx.Deserialize(bytes.NewReader(serializedTx)) + if err != nil { + str := "failed to deserialize transaction" + return nil, storeError(ErrInput, str, err) + } + copy(rec.Hash[:], wire.DoubleSha256(serializedTx)) + return rec, nil +} + +// NewTxRecordFromMsgTx creates a new transaction record that may be inserted +// into the store. +func NewTxRecordFromMsgTx(msgTx *wire.MsgTx, received time.Time) (*TxRecord, error) { + buf := bytes.NewBuffer(make([]byte, 0, msgTx.SerializeSize())) + err := msgTx.Serialize(buf) + if err != nil { + str := "failed to serialize transaction" + return nil, storeError(ErrInput, str, err) + } + rec := &TxRecord{ + MsgTx: *msgTx, + Received: received, + SerializedTx: buf.Bytes(), + } + copy(rec.Hash[:], wire.DoubleSha256(rec.SerializedTx)) + return rec, nil +} + +// Credit is the type representing a transaction output which was spent or +// is still spendable by wallet. A UTXO is an unspent Credit, but not all +// Credits are UTXOs. +type Credit struct { + wire.OutPoint + BlockMeta + Amount btcutil.Amount + PkScript []byte + Received time.Time + FromCoinBase bool +} + +// Store implements a transaction store for storing and managing wallet +// transactions. +type Store struct { + namespace walletdb.Namespace +} + +// Open opens the wallet transaction store from a walletdb namespace. If the +// store does not exist, ErrNoExist is returned. Existing stores will be +// upgraded to new database formats as necessary. +func Open(namespace walletdb.Namespace) (*Store, error) { + // Open the store, upgrading to the latest version as needed. + err := openStore(namespace) + if err != nil { + return nil, err + } + return &Store{namespace}, nil +} + +// Create creates and opens a new persistent transaction store in the walletdb +// namespace. Creating the store when one already exists in this namespace will +// error with ErrAlreadyExists. +func Create(namespace walletdb.Namespace) (*Store, error) { + err := createStore(namespace) + if err != nil { + return nil, err + } + return &Store{namespace}, nil +} + +// moveMinedTx moves a transaction record from the unmined buckets to block +// buckets. +func (s *Store) moveMinedTx(ns walletdb.Bucket, rec *TxRecord, recKey, recVal []byte, block *BlockMeta) error { + log.Infof("Marking unconfirmed transaction %v mined in block %d", + &rec.Hash, block.Height) + + // Insert block record as needed. + blockKey, blockVal := existsBlockRecord(ns, block.Height) + var err error + if blockVal == nil { + blockVal = valueBlockRecord(block, &rec.Hash) + } else { + blockVal, err = appendRawBlockRecord(blockVal, &rec.Hash) + if err != nil { + return err + } + } + err = putRawBlockRecord(ns, blockKey, blockVal) + if err != nil { + return err + } + + err = putRawTxRecord(ns, recKey, recVal) + if err != nil { + return err + } + minedBalance, err := fetchMinedBalance(ns) + if err != nil { + return err + } + + // For all mined transactions with unspent credits spent by this + // transaction, mark each spent, remove from the unspents map, and + // insert a debit record for the spent credit. + debitIncidence := indexedIncidence{ + incidence: incidence{txHash: rec.Hash, block: block.Block}, + // index set for each rec input below. + } + for i, input := range rec.MsgTx.TxIn { + unspentKey, credKey := existsUnspent(ns, &input.PreviousOutPoint) + if credKey == nil { + continue + } + debitIncidence.index = uint32(i) + amt, err := spendCredit(ns, credKey, &debitIncidence) + if err != nil { + return err + } + minedBalance -= amt + err = deleteRawUnspent(ns, unspentKey) + if err != nil { + return err + } + + err = putDebit(ns, &rec.Hash, uint32(i), amt, &block.Block, credKey) + if err != nil { + return err + } + + err = deleteRawUnminedInput(ns, unspentKey) + if err != nil { + return err + } + } + + // For each output of the record that is marked as a credit, if the + // output is marked as a credit by the unconfirmed store, remove the + // marker and mark the output as a credit in the db. + // + // Moved credits are added as unspents, even if there is another + // unconfirmed transaction which spends them. + cred := credit{ + outPoint: wire.OutPoint{Hash: rec.Hash}, + block: block.Block, + spentBy: indexedIncidence{index: ^uint32(0)}, + } + it := makeUnminedCreditIterator(ns, &rec.Hash) + for it.next() { + // TODO: This should use the raw apis. The credit value (it.cv) + // can be moved from unmined directly to the credits bucket. + // The key needs a modification to include the block + // height/hash. + index, err := fetchRawUnminedCreditIndex(it.ck) + if err != nil { + return err + } + amount, change, err := fetchRawUnminedCreditAmountChange(it.cv) + if err != nil { + return err + } + cred.outPoint.Index = index + cred.amount = amount + cred.change = change + + err = it.delete() + if err != nil { + return err + } + err = putUnspentCredit(ns, &cred) + if err != nil { + return err + } + err = putUnspent(ns, &cred.outPoint, &block.Block) + if err != nil { + return err + } + minedBalance += amount + } + if it.err != nil { + return it.err + } + + err = putMinedBalance(ns, minedBalance) + if err != nil { + return err + } + + return deleteRawUnmined(ns, rec.Hash[:]) +} + +// InsertTx records a transaction as belonging to a wallet's transaction +// history. If block is nil, the transaction is considered unspent, and the +// transaction's index must be unset. +func (s *Store) InsertTx(rec *TxRecord, block *BlockMeta) error { + return scopedUpdate(s.namespace, func(ns walletdb.Bucket) error { + if block == nil { + return s.insertMemPoolTx(ns, rec) + } + return s.insertMinedTx(ns, rec, block) + }) +} + +// insertMinedTx inserts a new transaction record for a mined transaction into +// the database. It is expected that the exact transation does not already +// exist in the unmined buckets, but unmined double spends (including mutations) +// are removed. +func (s *Store) insertMinedTx(ns walletdb.Bucket, rec *TxRecord, block *BlockMeta) error { + // If a transaction record for this tx hash and block already exist, + // there is nothing left to do. + k, v := existsTxRecord(ns, &rec.Hash, &block.Block) + if v != nil { + return nil + } + + // If the exact tx (not a double spend) is already included but + // unconfirmed, move it to a block. + v = existsRawUnmined(ns, rec.Hash[:]) + if v != nil { + return s.moveMinedTx(ns, rec, k, v, block) + } + + // As there may be unconfirmed transactions that are invalidated by this + // transaction (either being duplicates, or double spends), remove them + // from the unconfirmed set. This also handles removing unconfirmed + // transaction spend chains if any other unconfirmed transactions spend + // outputs of the removed double spend. + err := s.removeDoubleSpends(ns, rec) + if err != nil { + return err + } + + // If a block record does not yet exist for any transactions from this + // block, insert the record. Otherwise, update it by adding the + // transaction hash to the set of transactions from this block. + blockKey, blockValue := existsBlockRecord(ns, block.Height) + if blockValue == nil { + err = putBlockRecord(ns, block, &rec.Hash) + } else { + blockValue, err = appendRawBlockRecord(blockValue, &rec.Hash) + if err != nil { + return err + } + err = putRawBlockRecord(ns, blockKey, blockValue) + } + if err != nil { + return err + } + + err = putTxRecord(ns, rec, &block.Block) + if err != nil { + return err + } + + minedBalance, err := fetchMinedBalance(ns) + if err != nil { + return err + } + + // Add a debit record for each unspent credit spent by this tx. + spender := indexedIncidence{ + incidence: incidence{ + txHash: rec.Hash, + block: block.Block, + }, + // index set for each iteration below + } + for i, input := range rec.MsgTx.TxIn { + unspentKey, credKey := existsUnspent(ns, &input.PreviousOutPoint) + if credKey == nil { + // Debits for unmined transactions are not explicitly + // tracked. Instead, all previous outputs spent by any + // unmined transaction are added to a map for quick + // lookups when it must be checked whether a mined + // output is unspent or not. + // + // Tracking individual debits for unmined transactions + // could be added later to simplify (and increase + // performance of) determining some details that need + // the previous outputs (e.g. determining a fee), but at + // the moment that is not done (and a db lookup is used + // for those cases instead). There is also a good + // chance that all unmined transaction handling will + // move entirely to the db rather than being handled in + // memory for atomicity reasons, so the simplist + // implementation is currently used. + continue + } + spender.index = uint32(i) + amt, err := spendCredit(ns, credKey, &spender) + if err != nil { + return err + } + err = putDebit(ns, &rec.Hash, uint32(i), amt, &block.Block, + credKey) + if err != nil { + return err + } + + minedBalance -= amt + + err = deleteRawUnspent(ns, unspentKey) + if err != nil { + return err + } + } + + return putMinedBalance(ns, minedBalance) +} + +// AddCredit marks a transaction record as containing a transaction output +// spendable by wallet. The output is added unspent, and is marked spent +// when a new transaction spending the output is inserted into the store. +// +// TODO(jrick): This should not be necessary. Instead, pass the indexes +// that are known to contain credits when a transaction or merkleblock is +// inserted into the store. +func (s *Store) AddCredit(rec *TxRecord, block *BlockMeta, index uint32, change bool) error { + if int(index) >= len(rec.MsgTx.TxOut) { + str := "transaction output does not exist" + return storeError(ErrInput, str, nil) + } + + return scopedUpdate(s.namespace, func(ns walletdb.Bucket) error { + return s.addCredit(ns, rec, block, index, change) + }) +} + +func (s *Store) addCredit(ns walletdb.Bucket, rec *TxRecord, block *BlockMeta, index uint32, change bool) error { + if block == nil { + k := canonicalOutPoint(&rec.Hash, index) + v := valueUnminedCredit(btcutil.Amount(rec.MsgTx.TxOut[index].Value), change) + return putRawUnminedCredit(ns, k, v) + } + + k, v := existsCredit(ns, &rec.Hash, index, &block.Block) + if v != nil { + return nil + } + + txOutAmt := btcutil.Amount(rec.MsgTx.TxOut[index].Value) + log.Debugf("Marking transaction %v output %d (%v) spendable", + rec.Hash, index, txOutAmt) + + cred := credit{ + outPoint: wire.OutPoint{ + Hash: rec.Hash, + Index: index, + }, + block: block.Block, + amount: txOutAmt, + change: change, + spentBy: indexedIncidence{index: ^uint32(0)}, + } + v = valueUnspentCredit(&cred) + err := putRawCredit(ns, k, v) + if err != nil { + return err + } + + minedBalance, err := fetchMinedBalance(ns) + if err != nil { + return err + } + err = putMinedBalance(ns, minedBalance+txOutAmt) + if err != nil { + return err + } + + return putUnspent(ns, &cred.outPoint, &block.Block) +} + +// Rollback removes all blocks at height onwards, moving any transactions within +// each block to the unconfirmed pool. +func (s *Store) Rollback(height int32) error { + return scopedUpdate(s.namespace, func(ns walletdb.Bucket) error { + return s.rollback(ns, height) + }) +} + +func (s *Store) rollback(ns walletdb.Bucket, height int32) error { + minedBalance, err := fetchMinedBalance(ns) + if err != nil { + return err + } + + // Keep track of all credits that were removed from coinbase + // transactions. After detaching all blocks, if any transaction record + // exists in unmined that spends these outputs, remove them and their + // spend chains. + // + // It is necessary to keep these in memory and fix the unmined + // transactions later since blocks are removed in increasing order. + var coinBaseCredits []wire.OutPoint + + it := makeBlockIterator(ns, height) + for it.next() { + b := &it.elem + + log.Infof("Rolling back %d transactions from block %v height %d", + len(b.transactions), b.Hash, b.Height) + + for i := range b.transactions { + txHash := &b.transactions[i] + + recKey := keyTxRecord(txHash, &b.Block) + recVal := existsRawTxRecord(ns, recKey) + var rec TxRecord + err = readRawTxRecord(txHash, recVal, &rec) + if err != nil { + return err + } + + err = deleteTxRecord(ns, txHash, &b.Block) + if err != nil { + return err + } + + // Handle coinbase transactions specially since they are + // not moved to the unconfirmed store. A coinbase cannot + // contain any debits, but all credits should be removed + // and the mined balance decremented. + if blockchain.IsCoinBaseTx(&rec.MsgTx) { + op := wire.OutPoint{Hash: rec.Hash} + for i, output := range rec.MsgTx.TxOut { + k, v := existsCredit(ns, &rec.Hash, + uint32(i), &b.Block) + if v == nil { + continue + } + op.Index = uint32(i) + + coinBaseCredits = append(coinBaseCredits, op) + + unspentKey, credKey := existsUnspent(ns, &op) + if credKey != nil { + minedBalance -= btcutil.Amount(output.Value) + err = deleteRawUnspent(ns, unspentKey) + if err != nil { + return err + } + } + err = deleteRawCredit(ns, k) + if err != nil { + return err + } + } + + continue + } + + err = putRawUnmined(ns, txHash[:], recVal) + if err != nil { + return err + } + + // For each debit recorded for this transaction, mark + // the credit it spends as unspent (as long as it still + // exists) and delete the debit. The previous output is + // recorded in the unconfirmed store for every previous + // output, not just debits. + for i, input := range rec.MsgTx.TxIn { + prevOut := &input.PreviousOutPoint + prevOutKey := canonicalOutPoint(&prevOut.Hash, + prevOut.Index) + err = putRawUnminedInput(ns, prevOutKey, rec.Hash[:]) + if err != nil { + return err + } + + // If this input is a debit, remove the debit + // record and mark the credit that it spent as + // unspent, incrementing the mined balance. + debKey, credKey, err := existsDebit(ns, + &rec.Hash, uint32(i), &b.Block) + if err != nil { + return err + } + if debKey == nil { + continue + } + + // unspendRawCredit does not error in case the + // no credit exists for this key, but this + // behavior is correct. Since blocks are + // removed in increasing order, this credit + // may have already been removed from a + // previously removed transaction record in + // this rollback. + var amt btcutil.Amount + amt, err = unspendRawCredit(ns, credKey) + if err != nil { + return err + } + err = deleteRawDebit(ns, debKey) + if err != nil { + return err + } + + // If the credit was previously removed in the + // rollback, the credit amount is zero. Only + // mark the previously spent credit as unspent + // if it still exists. + if amt == 0 { + continue + } + unspentVal, err := fetchRawCreditUnspentValue(credKey) + if err != nil { + return err + } + minedBalance += amt + err = putRawUnspent(ns, prevOutKey, unspentVal) + if err != nil { + return err + } + } + + // For each detached non-coinbase credit, move the + // credit output to unmined. If the credit is marked + // unspent, it is removed from the utxo set and the + // mined balance is decremented. + // + // TODO: use a credit iterator + for i, output := range rec.MsgTx.TxOut { + k, v := existsCredit(ns, &rec.Hash, uint32(i), + &b.Block) + if v == nil { + continue + } + + amt, change, err := fetchRawCreditAmountChange(v) + if err != nil { + return err + } + outPointKey := canonicalOutPoint(&rec.Hash, uint32(i)) + unminedCredVal := valueUnminedCredit(amt, change) + err = putRawUnminedCredit(ns, outPointKey, unminedCredVal) + if err != nil { + return err + } + + err = deleteRawCredit(ns, k) + if err != nil { + return err + } + + credKey := existsRawUnspent(ns, outPointKey) + if credKey != nil { + minedBalance -= btcutil.Amount(output.Value) + err = deleteRawUnspent(ns, outPointKey) + if err != nil { + return err + } + } + } + } + + err = it.delete() + if err != nil { + return err + } + } + if it.err != nil { + return it.err + } + + for _, op := range coinBaseCredits { + opKey := canonicalOutPoint(&op.Hash, op.Index) + unminedKey := existsRawUnminedInput(ns, opKey) + if unminedKey != nil { + unminedVal := existsRawUnmined(ns, unminedKey) + var unminedRec TxRecord + copy(unminedRec.Hash[:], unminedKey) // Silly but need an array + err = readRawTxRecord(&unminedRec.Hash, unminedVal, &unminedRec) + if err != nil { + return err + } + + log.Debugf("Transaction %v spends a removed coinbase "+ + "output -- removing as well", unminedRec.Hash) + err = s.removeConflict(ns, &unminedRec) + if err != nil { + return err + } + } + } + + return putMinedBalance(ns, minedBalance) +} + +// UnspentOutputs returns all unspent received transaction outputs. +// The order is undefined. +func (s *Store) UnspentOutputs() ([]Credit, error) { + var credits []Credit + err := scopedView(s.namespace, func(ns walletdb.Bucket) error { + var err error + credits, err = s.unspentOutputs(ns) + return err + }) + return credits, err +} + +func (s *Store) unspentOutputs(ns walletdb.Bucket) ([]Credit, error) { + var unspent []Credit + + var op wire.OutPoint + var block Block + err := ns.Bucket(bucketUnspent).ForEach(func(k, v []byte) error { + err := readCanonicalOutPoint(k, &op) + if err != nil { + return err + } + if existsRawUnminedInput(ns, k) != nil { + // Output is spent by an unmined transaction. + // Skip this k/v pair. + return nil + } + err = readUnspentBlock(v, &block) + if err != nil { + return err + } + + blockTime, err := fetchBlockTime(ns, block.Height) + if err != nil { + return err + } + // TODO(jrick): reading the entire transaction should + // be avoidable. Creating the credit only requires the + // output amount and pkScript. + rec, err := fetchTxRecord(ns, &op.Hash, &block) + if err != nil { + return err + } + txOut := rec.MsgTx.TxOut[op.Index] + cred := Credit{ + OutPoint: op, + BlockMeta: BlockMeta{ + Block: block, + Time: blockTime, + }, + Amount: btcutil.Amount(txOut.Value), + PkScript: txOut.PkScript, + Received: rec.Received, + FromCoinBase: blockchain.IsCoinBaseTx(&rec.MsgTx), + } + unspent = append(unspent, cred) + return nil + }) + if err != nil { + if _, ok := err.(Error); ok { + return nil, err + } + str := "failed iterating unspent bucket" + return nil, storeError(ErrDatabase, str, err) + } + + err = ns.Bucket(bucketUnminedCredits).ForEach(func(k, v []byte) error { + if existsRawUnminedInput(ns, k) != nil { + // Output is spent by an unmined transaction. + // Skip to next unmined credit. + return nil + } + + err := readCanonicalOutPoint(k, &op) + if err != nil { + return err + } + + // TODO(jrick): Reading/parsing the entire transaction record + // just for the output amount and script can be avoided. + recVal := existsRawUnmined(ns, op.Hash[:]) + var rec TxRecord + err = readRawTxRecord(&op.Hash, recVal, &rec) + if err != nil { + return err + } + + txOut := rec.MsgTx.TxOut[op.Index] + cred := Credit{ + OutPoint: op, + BlockMeta: BlockMeta{ + Block: Block{Height: -1}, + }, + Amount: btcutil.Amount(txOut.Value), + PkScript: txOut.PkScript, + Received: rec.Received, + FromCoinBase: blockchain.IsCoinBaseTx(&rec.MsgTx), + } + unspent = append(unspent, cred) + return nil + }) + if err != nil { + if _, ok := err.(Error); ok { + return nil, err + } + str := "failed iterating unmined credits bucket" + return nil, storeError(ErrDatabase, str, err) + } + + return unspent, nil +} + +// Balance returns the spendable wallet balance (total value of all unspent +// transaction outputs) given a minimum of minConf confirmations, calculated +// at a current chain height of curHeight. Coinbase outputs are only included +// in the balance if maturity has been reached. +// +// Balance may return unexpected results if syncHeight is lower than the block +// height of the most recent mined transaction in the store. +func (s *Store) Balance(minConf, syncHeight int32) (btcutil.Amount, error) { + var amt btcutil.Amount + err := scopedView(s.namespace, func(ns walletdb.Bucket) error { + var err error + amt, err = s.balance(ns, minConf, syncHeight) + return err + }) + return amt, err +} + +func (s *Store) balance(ns walletdb.Bucket, minConf int32, syncHeight int32) (btcutil.Amount, error) { + bal, err := fetchMinedBalance(ns) + if err != nil { + return 0, err + } + + // Subtract the balance for each credit that is spent by an unmined + // transaction. + var op wire.OutPoint + var block Block + err = ns.Bucket(bucketUnspent).ForEach(func(k, v []byte) error { + err := readCanonicalOutPoint(k, &op) + if err != nil { + return err + } + err = readUnspentBlock(v, &block) + if err != nil { + return err + } + if existsRawUnminedInput(ns, k) != nil { + _, v := existsCredit(ns, &op.Hash, op.Index, &block) + amt, err := fetchRawCreditAmount(v) + if err != nil { + return err + } + bal -= amt + } + return nil + }) + if err != nil { + if _, ok := err.(Error); ok { + return 0, err + } + str := "failed iterating unspent outputs" + return 0, storeError(ErrDatabase, str, err) + } + + // Decrement the balance for any unspent credit with less than + // minConf confirmations and any (unspent) immature coinbase credit. + stopConf := minConf + if blockchain.CoinbaseMaturity > stopConf { + stopConf = blockchain.CoinbaseMaturity + } + lastHeight := syncHeight - stopConf + blockIt := makeReverseBlockIterator(ns) + for blockIt.prev() { + block := &blockIt.elem + + if block.Height < lastHeight { + break + } + + for i := range block.transactions { + txHash := &block.transactions[i] + rec, err := fetchTxRecord(ns, txHash, &block.Block) + if err != nil { + return 0, err + } + numOuts := uint32(len(rec.MsgTx.TxOut)) + for i := uint32(0); i < numOuts; i++ { + // Avoid double decrementing the credit amount + // if it was already removed for being spent by + // an unmined tx. + opKey := canonicalOutPoint(txHash, i) + if existsRawUnminedInput(ns, opKey) != nil { + continue + } + + _, v := existsCredit(ns, txHash, i, &block.Block) + if v == nil { + continue + } + amt, spent, err := fetchRawCreditAmountSpent(v) + if err != nil { + return 0, err + } + if spent { + continue + } + confs := syncHeight - block.Height + 1 + if confs < minConf || (blockchain.IsCoinBaseTx(&rec.MsgTx) && + confs < blockchain.CoinbaseMaturity) { + bal -= amt + } + } + } + } + if blockIt.err != nil { + return 0, blockIt.err + } + + // If unmined outputs are included, increment the balance for each + // output that is unspent. + if minConf == 0 { + err = ns.Bucket(bucketUnminedCredits).ForEach(func(k, v []byte) error { + if existsRawUnminedInput(ns, k) != nil { + // Output is spent by an unmined transaction. + // Skip to next unmined credit. + return nil + } + + amount, err := fetchRawUnminedCreditAmount(v) + if err != nil { + return err + } + bal += amount + return nil + }) + if err != nil { + if _, ok := err.(Error); ok { + return 0, err + } + str := "failed to iterate over unmined credits bucket" + return 0, storeError(ErrDatabase, str, err) + } + } + + return bal, nil +} diff --git a/wtxmgr/tx_test.go b/wtxmgr/tx_test.go new file mode 100644 index 0000000..78f228c --- /dev/null +++ b/wtxmgr/tx_test.go @@ -0,0 +1,1213 @@ +// Copyright (c) 2013-2015 Conformal Systems LLC +// +// Permission to use, copy, modify, and distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +package wtxmgr_test + +import ( + "bytes" + "encoding/hex" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwallet/walletdb" + _ "github.com/btcsuite/btcwallet/walletdb/bdb" + . "github.com/btcsuite/btcwallet/wtxmgr" +) + +// Received transaction output for mainnet outpoint +// 61d3696de4c888730cbe06b0ad8ecb6d72d6108e893895aa9bc067bd7eba3fad:0 +var ( + TstRecvSerializedTx, _ = hex.DecodeString("010000000114d9ff358894c486b4ae11c2a8cf7851b1df64c53d2e511278eff17c22fb7373000000008c493046022100995447baec31ee9f6d4ec0e05cb2a44f6b817a99d5f6de167d1c75354a946410022100c9ffc23b64d770b0e01e7ff4d25fbc2f1ca8091053078a247905c39fce3760b601410458b8e267add3c1e374cf40f1de02b59213a82e1d84c2b94096e22e2f09387009c96debe1d0bcb2356ffdcf65d2a83d4b34e72c62eccd8490dbf2110167783b2bffffffff0280969800000000001976a914479ed307831d0ac19ebc5f63de7d5f1a430ddb9d88ac38bfaa00000000001976a914dadf9e3484f28b385ddeaa6c575c0c0d18e9788a88ac00000000") + TstRecvTx, _ = btcutil.NewTxFromBytes(TstRecvSerializedTx) + TstRecvTxSpendingTxBlockHash, _ = wire.NewShaHashFromStr("00000000000000017188b968a371bab95aa43522665353b646e41865abae02a4") + TstRecvAmt = int64(10000000) + TstRecvTxBlockDetails = &BlockMeta{ + Block: Block{Hash: *TstRecvTxSpendingTxBlockHash, Height: 276425}, + Time: time.Unix(1387737310, 0), + } + + TstRecvCurrentHeight = int32(284498) // mainnet blockchain height at time of writing + TstRecvTxOutConfirms = 8074 // hardcoded number of confirmations given the above block height + + TstSpendingSerializedTx, _ = hex.DecodeString("0100000003ad3fba7ebd67c09baa9538898e10d6726dcb8eadb006be0c7388c8e46d69d361000000006b4830450220702c4fbde5532575fed44f8d6e8c3432a2a9bd8cff2f966c3a79b2245a7c88db02210095d6505a57e350720cb52b89a9b56243c15ddfcea0596aedc1ba55d9fb7d5aa0012103cccb5c48a699d3efcca6dae277fee6b82e0229ed754b742659c3acdfed2651f9ffffffffdbd36173f5610e34de5c00ed092174603761595d90190f790e79cda3e5b45bc2010000006b483045022000fa20735e5875e64d05bed43d81b867f3bd8745008d3ff4331ef1617eac7c44022100ad82261fc57faac67fc482a37b6bf18158da0971e300abf5fe2f9fd39e107f58012102d4e1caf3e022757512c204bf09ff56a9981df483aba3c74bb60d3612077c9206ffffffff65536c9d964b6f89b8ef17e83c6666641bc495cb27bab60052f76cd4556ccd0d040000006a473044022068e3886e0299ffa69a1c3ee40f8b6700f5f6d463a9cf9dbf22c055a131fc4abc02202b58957fe19ff1be7a84c458d08016c53fbddec7184ac5e633f2b282ae3420ae012103b4e411b81d32a69fb81178a8ea1abaa12f613336923ee920ffbb1b313af1f4d2ffffffff02ab233200000000001976a91418808b2fbd8d2c6d022aed5cd61f0ce6c0a4cbb688ac4741f011000000001976a914f081088a300c80ce36b717a9914ab5ec8a7d283988ac00000000") + TstSpendingTx, _ = btcutil.NewTxFromBytes(TstSpendingSerializedTx) + TstSpendingTxBlockHeight = int32(279143) + TstSignedTxBlockHash, _ = wire.NewShaHashFromStr("00000000000000017188b968a371bab95aa43522665353b646e41865abae02a4") + TstSignedTxBlockDetails = &BlockMeta{ + Block: Block{Hash: *TstSignedTxBlockHash, Height: TstSpendingTxBlockHeight}, + Time: time.Unix(1389114091, 0), + } +) + +func testDB() (walletdb.DB, func(), error) { + tmpDir, err := ioutil.TempDir("", "wtxmgr_test") + if err != nil { + return nil, func() {}, err + } + db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db")) + return db, func() { os.RemoveAll(tmpDir) }, err +} + +func testStore() (*Store, func(), error) { + tmpDir, err := ioutil.TempDir("", "wtxmgr_test") + if err != nil { + return nil, func() {}, err + } + db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db")) + if err != nil { + teardown := func() { + os.RemoveAll(tmpDir) + } + return nil, teardown, err + } + teardown := func() { + db.Close() + os.RemoveAll(tmpDir) + } + ns, err := db.Namespace([]byte("txstore")) + if err != nil { + return nil, teardown, err + } + s, err := Create(ns) + return s, teardown, err +} + +func serializeTx(tx *btcutil.Tx) []byte { + var buf bytes.Buffer + err := tx.MsgTx().Serialize(&buf) + if err != nil { + panic(err) + } + return buf.Bytes() +} + +func TestInsertsCreditsDebitsRollbacks(t *testing.T) { + t.Parallel() + + // Create a double spend of the received blockchain transaction. + dupRecvTx, _ := btcutil.NewTxFromBytes(TstRecvSerializedTx) + // Switch txout amount to 1 BTC. Transaction store doesn't + // validate txs, so this is fine for testing a double spend + // removal. + TstDupRecvAmount := int64(1e8) + newDupMsgTx := dupRecvTx.MsgTx() + newDupMsgTx.TxOut[0].Value = TstDupRecvAmount + TstDoubleSpendTx := btcutil.NewTx(newDupMsgTx) + TstDoubleSpendSerializedTx := serializeTx(TstDoubleSpendTx) + + // Create a "signed" (with invalid sigs) tx that spends output 0 of + // the double spend. + spendingTx := wire.NewMsgTx() + spendingTxIn := wire.NewTxIn(wire.NewOutPoint(TstDoubleSpendTx.Sha(), 0), []byte{0, 1, 2, 3, 4}) + spendingTx.AddTxIn(spendingTxIn) + spendingTxOut1 := wire.NewTxOut(1e7, []byte{5, 6, 7, 8, 9}) + spendingTxOut2 := wire.NewTxOut(9e7, []byte{10, 11, 12, 13, 14}) + spendingTx.AddTxOut(spendingTxOut1) + spendingTx.AddTxOut(spendingTxOut2) + TstSpendingTx := btcutil.NewTx(spendingTx) + TstSpendingSerializedTx := serializeTx(TstSpendingTx) + var _ = TstSpendingTx + + tests := []struct { + name string + f func(*Store) (*Store, error) + bal, unc btcutil.Amount + unspents map[wire.OutPoint]struct{} + unmined map[wire.ShaHash]struct{} + }{ + { + name: "new store", + f: func(s *Store) (*Store, error) { + return s, nil + }, + bal: 0, + unc: 0, + unspents: map[wire.OutPoint]struct{}{}, + unmined: map[wire.ShaHash]struct{}{}, + }, + { + name: "txout insert", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstRecvSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, nil) + if err != nil { + return nil, err + } + + err = s.AddCredit(rec, nil, 0, false) + return s, err + }, + bal: 0, + unc: btcutil.Amount(TstRecvTx.MsgTx().TxOut[0].Value), + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstRecvTx.Sha(), 0}: {}, + }, + unmined: map[wire.ShaHash]struct{}{ + *TstRecvTx.Sha(): {}, + }, + }, + { + name: "insert duplicate unconfirmed", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstRecvSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, nil) + if err != nil { + return nil, err + } + + err = s.AddCredit(rec, nil, 0, false) + return s, err + }, + bal: 0, + unc: btcutil.Amount(TstRecvTx.MsgTx().TxOut[0].Value), + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstRecvTx.Sha(), 0}: {}, + }, + unmined: map[wire.ShaHash]struct{}{ + *TstRecvTx.Sha(): {}, + }, + }, + { + name: "confirmed txout insert", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstRecvSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, TstRecvTxBlockDetails) + if err != nil { + return nil, err + } + + err = s.AddCredit(rec, TstRecvTxBlockDetails, 0, false) + return s, err + }, + bal: btcutil.Amount(TstRecvTx.MsgTx().TxOut[0].Value), + unc: 0, + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstRecvTx.Sha(), 0}: {}, + }, + unmined: map[wire.ShaHash]struct{}{}, + }, + { + name: "insert duplicate confirmed", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstRecvSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, TstRecvTxBlockDetails) + if err != nil { + return nil, err + } + + err = s.AddCredit(rec, TstRecvTxBlockDetails, 0, false) + return s, err + }, + bal: btcutil.Amount(TstRecvTx.MsgTx().TxOut[0].Value), + unc: 0, + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstRecvTx.Sha(), 0}: {}, + }, + unmined: map[wire.ShaHash]struct{}{}, + }, + { + name: "rollback confirmed credit", + f: func(s *Store) (*Store, error) { + err := s.Rollback(TstRecvTxBlockDetails.Height) + return s, err + }, + bal: 0, + unc: btcutil.Amount(TstRecvTx.MsgTx().TxOut[0].Value), + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstRecvTx.Sha(), 0}: {}, + }, + unmined: map[wire.ShaHash]struct{}{ + *TstRecvTx.Sha(): {}, + }, + }, + { + name: "insert confirmed double spend", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstDoubleSpendSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, TstRecvTxBlockDetails) + if err != nil { + return nil, err + } + + err = s.AddCredit(rec, TstRecvTxBlockDetails, 0, false) + return s, err + }, + bal: btcutil.Amount(TstDoubleSpendTx.MsgTx().TxOut[0].Value), + unc: 0, + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstDoubleSpendTx.Sha(), 0}: {}, + }, + unmined: map[wire.ShaHash]struct{}{}, + }, + { + name: "insert unconfirmed debit", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstSpendingSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, nil) + return s, err + }, + bal: 0, + unc: 0, + unspents: map[wire.OutPoint]struct{}{}, + unmined: map[wire.ShaHash]struct{}{ + *TstSpendingTx.Sha(): {}, + }, + }, + { + name: "insert unconfirmed debit again", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstDoubleSpendSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, TstRecvTxBlockDetails) + return s, err + }, + bal: 0, + unc: 0, + unspents: map[wire.OutPoint]struct{}{}, + unmined: map[wire.ShaHash]struct{}{ + *TstSpendingTx.Sha(): {}, + }, + }, + { + name: "insert change (index 0)", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstSpendingSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, nil) + if err != nil { + return nil, err + } + + err = s.AddCredit(rec, nil, 0, true) + return s, err + }, + bal: 0, + unc: btcutil.Amount(TstSpendingTx.MsgTx().TxOut[0].Value), + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstSpendingTx.Sha(), 0}: {}, + }, + unmined: map[wire.ShaHash]struct{}{ + *TstSpendingTx.Sha(): {}, + }, + }, + { + name: "insert output back to this own wallet (index 1)", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstSpendingSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, nil) + if err != nil { + return nil, err + } + err = s.AddCredit(rec, nil, 1, true) + return s, err + }, + bal: 0, + unc: btcutil.Amount(TstSpendingTx.MsgTx().TxOut[0].Value + TstSpendingTx.MsgTx().TxOut[1].Value), + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstSpendingTx.Sha(), 0}: {}, + wire.OutPoint{*TstSpendingTx.Sha(), 1}: {}, + }, + unmined: map[wire.ShaHash]struct{}{ + *TstSpendingTx.Sha(): {}, + }, + }, + { + name: "confirm signed tx", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstSpendingSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, TstSignedTxBlockDetails) + return s, err + }, + bal: btcutil.Amount(TstSpendingTx.MsgTx().TxOut[0].Value + TstSpendingTx.MsgTx().TxOut[1].Value), + unc: 0, + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstSpendingTx.Sha(), 0}: {}, + wire.OutPoint{*TstSpendingTx.Sha(), 1}: {}, + }, + unmined: map[wire.ShaHash]struct{}{}, + }, + { + name: "rollback after spending tx", + f: func(s *Store) (*Store, error) { + err := s.Rollback(TstSignedTxBlockDetails.Height + 1) + return s, err + }, + bal: btcutil.Amount(TstSpendingTx.MsgTx().TxOut[0].Value + TstSpendingTx.MsgTx().TxOut[1].Value), + unc: 0, + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstSpendingTx.Sha(), 0}: {}, + wire.OutPoint{*TstSpendingTx.Sha(), 1}: {}, + }, + unmined: map[wire.ShaHash]struct{}{}, + }, + { + name: "rollback spending tx block", + f: func(s *Store) (*Store, error) { + err := s.Rollback(TstSignedTxBlockDetails.Height) + return s, err + }, + bal: 0, + unc: btcutil.Amount(TstSpendingTx.MsgTx().TxOut[0].Value + TstSpendingTx.MsgTx().TxOut[1].Value), + unspents: map[wire.OutPoint]struct{}{ + wire.OutPoint{*TstSpendingTx.Sha(), 0}: {}, + wire.OutPoint{*TstSpendingTx.Sha(), 1}: {}, + }, + unmined: map[wire.ShaHash]struct{}{ + *TstSpendingTx.Sha(): {}, + }, + }, + { + name: "rollback double spend tx block", + f: func(s *Store) (*Store, error) { + err := s.Rollback(TstRecvTxBlockDetails.Height) + return s, err + }, + bal: 0, + unc: btcutil.Amount(TstSpendingTx.MsgTx().TxOut[0].Value + TstSpendingTx.MsgTx().TxOut[1].Value), + unspents: map[wire.OutPoint]struct{}{ + *wire.NewOutPoint(TstSpendingTx.Sha(), 0): {}, + *wire.NewOutPoint(TstSpendingTx.Sha(), 1): {}, + }, + unmined: map[wire.ShaHash]struct{}{ + *TstDoubleSpendTx.Sha(): {}, + *TstSpendingTx.Sha(): {}, + }, + }, + { + name: "insert original recv txout", + f: func(s *Store) (*Store, error) { + rec, err := NewTxRecord(TstRecvSerializedTx, time.Now()) + if err != nil { + return nil, err + } + err = s.InsertTx(rec, TstRecvTxBlockDetails) + if err != nil { + return nil, err + } + err = s.AddCredit(rec, TstRecvTxBlockDetails, 0, false) + return s, err + }, + bal: btcutil.Amount(TstRecvTx.MsgTx().TxOut[0].Value), + unc: 0, + unspents: map[wire.OutPoint]struct{}{ + *wire.NewOutPoint(TstRecvTx.Sha(), 0): {}, + }, + unmined: map[wire.ShaHash]struct{}{}, + }, + } + + s, teardown, err := testStore() + defer teardown() + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + tmpStore, err := test.f(s) + if err != nil { + t.Fatalf("%s: got error: %v", test.name, err) + } + s = tmpStore + bal, err := s.Balance(1, TstRecvCurrentHeight) + if err != nil { + t.Fatalf("%s: Confirmed Balance failed: %v", test.name, err) + } + if bal != test.bal { + t.Fatalf("%s: balance mismatch: expected: %d, got: %d", test.name, test.bal, bal) + } + unc, err := s.Balance(0, TstRecvCurrentHeight) + if err != nil { + t.Fatalf("%s: Unconfirmed Balance failed: %v", test.name, err) + } + unc -= bal + if unc != test.unc { + t.Fatalf("%s: unconfirmed balance mismatch: expected %d, got %d", test.name, test.unc, unc) + } + + // Check that unspent outputs match expected. + unspent, err := s.UnspentOutputs() + if err != nil { + t.Fatalf("%s: failed to fetch unspent outputs: %v", test.name, err) + } + for _, cred := range unspent { + if _, ok := test.unspents[cred.OutPoint]; !ok { + t.Errorf("%s: unexpected unspent output: %v", test.name, cred.OutPoint) + } + delete(test.unspents, cred.OutPoint) + } + if len(test.unspents) != 0 { + t.Fatalf("%s: missing expected unspent output(s)", test.name) + } + + // Check that unmined txs match expected. + unmined, err := s.UnminedTxs() + if err != nil { + t.Fatalf("%s: cannot load unmined transactions: %v", test.name, err) + } + for _, tx := range unmined { + txHash := tx.TxSha() + if _, ok := test.unmined[txHash]; !ok { + t.Fatalf("%s: unexpected unmined tx: %v", test.name, txHash) + } + delete(test.unmined, txHash) + } + if len(test.unmined) != 0 { + t.Fatalf("%s: missing expected unmined tx(s)", test.name) + } + + } +} + +func TestFindingSpentCredits(t *testing.T) { + t.Parallel() + + s, teardown, err := testStore() + defer teardown() + if err != nil { + t.Fatal(err) + } + + // Insert transaction and credit which will be spent. + recvRec, err := NewTxRecord(TstRecvSerializedTx, time.Now()) + if err != nil { + t.Fatal(err) + } + + err = s.InsertTx(recvRec, TstRecvTxBlockDetails) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(recvRec, TstRecvTxBlockDetails, 0, false) + if err != nil { + t.Fatal(err) + } + + // Insert confirmed transaction which spends the above credit. + spendingRec, err := NewTxRecord(TstSpendingSerializedTx, time.Now()) + if err != nil { + t.Fatal(err) + } + + err = s.InsertTx(spendingRec, TstSignedTxBlockDetails) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(spendingRec, TstSignedTxBlockDetails, 0, false) + if err != nil { + t.Fatal(err) + } + + bal, err := s.Balance(1, TstSignedTxBlockDetails.Height) + if err != nil { + t.Fatal(err) + } + expectedBal := btcutil.Amount(TstSpendingTx.MsgTx().TxOut[0].Value) + if bal != expectedBal { + t.Fatalf("bad balance: %v != %v", bal, expectedBal) + } + unspents, err := s.UnspentOutputs() + if err != nil { + t.Fatal(err) + } + op := wire.NewOutPoint(TstSpendingTx.Sha(), 0) + if unspents[0].OutPoint != *op { + t.Fatal("unspent outpoint doesn't match expected") + } + if len(unspents) > 1 { + t.Fatal("has more than one unspent credit") + } +} + +func newCoinBase(outputValues ...int64) *wire.MsgTx { + tx := wire.MsgTx{ + TxIn: []*wire.TxIn{ + &wire.TxIn{ + PreviousOutPoint: wire.OutPoint{Index: ^uint32(0)}, + }, + }, + } + for _, val := range outputValues { + tx.TxOut = append(tx.TxOut, &wire.TxOut{Value: val}) + } + return &tx +} + +func spendOutput(txHash *wire.ShaHash, index uint32, outputValues ...int64) *wire.MsgTx { + tx := wire.MsgTx{ + TxIn: []*wire.TxIn{ + &wire.TxIn{ + PreviousOutPoint: wire.OutPoint{Hash: *txHash, Index: index}, + }, + }, + } + for _, val := range outputValues { + tx.TxOut = append(tx.TxOut, &wire.TxOut{Value: val}) + } + return &tx +} + +func TestCoinbases(t *testing.T) { + t.Parallel() + + s, teardown, err := testStore() + defer teardown() + if err != nil { + t.Fatal(err) + } + + b100 := BlockMeta{ + Block: Block{Height: 100}, + Time: time.Now(), + } + + cb := newCoinBase(20e8, 10e8, 30e8) + cbRec, err := NewTxRecordFromMsgTx(cb, b100.Time) + if err != nil { + t.Fatal(err) + } + + // Insert coinbase and mark outputs 0 and 2 as credits. + err = s.InsertTx(cbRec, &b100) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(cbRec, &b100, 0, false) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(cbRec, &b100, 2, false) + if err != nil { + t.Fatal(err) + } + + // Balance should be 0 if the coinbase is immature, 50 BTC at and beyond + // maturity. + // + // Outputs when depth is below maturity are never included, no matter + // the required number of confirmations. Matured outputs which have + // greater depth than minConf are still excluded. + type balTest struct { + height int32 + minConf int32 + bal btcutil.Amount + } + balTests := []balTest{ + // Next block it is still immature + { + height: b100.Height + blockchain.CoinbaseMaturity - 2, + minConf: 0, + bal: 0, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 2, + minConf: blockchain.CoinbaseMaturity, + bal: 0, + }, + + // Next block it matures + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: 0, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: 1, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: blockchain.CoinbaseMaturity - 1, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: blockchain.CoinbaseMaturity, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: blockchain.CoinbaseMaturity + 1, + bal: 0, + }, + + // Matures at this block + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: 0, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: 1, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: blockchain.CoinbaseMaturity, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: blockchain.CoinbaseMaturity + 1, + bal: 50e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: blockchain.CoinbaseMaturity + 2, + bal: 0, + }, + } + for i, tst := range balTests { + bal, err := s.Balance(tst.minConf, tst.height) + if err != nil { + t.Fatalf("Balance test %d: Store.Balance failed: %v", i, err) + } + if bal != tst.bal { + t.Errorf("Balance test %d: Got %v Expected %v", i, bal, tst.bal) + } + } + if t.Failed() { + t.Fatal("Failed balance checks after inserting coinbase") + } + + // Spend an output from the coinbase tx in an unmined transaction when + // the next block will mature the coinbase. + spenderATime := time.Now() + spenderA := spendOutput(&cbRec.Hash, 0, 5e8, 15e8) + spenderARec, err := NewTxRecordFromMsgTx(spenderA, spenderATime) + if err != nil { + t.Fatal(err) + } + err = s.InsertTx(spenderARec, nil) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(spenderARec, nil, 0, false) + if err != nil { + t.Fatal(err) + } + + balTests = []balTest{ + // Next block it matures + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: 0, + bal: 35e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: 1, + bal: 30e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: blockchain.CoinbaseMaturity, + bal: 30e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity - 1, + minConf: blockchain.CoinbaseMaturity + 1, + bal: 0, + }, + + // Matures at this block + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: 0, + bal: 35e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: 1, + bal: 30e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: blockchain.CoinbaseMaturity, + bal: 30e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: blockchain.CoinbaseMaturity + 1, + bal: 30e8, + }, + { + height: b100.Height + blockchain.CoinbaseMaturity, + minConf: blockchain.CoinbaseMaturity + 2, + bal: 0, + }, + } + balTestsBeforeMaturity := balTests + for i, tst := range balTests { + bal, err := s.Balance(tst.minConf, tst.height) + if err != nil { + t.Fatalf("Balance test %d: Store.Balance failed: %v", i, err) + } + if bal != tst.bal { + t.Errorf("Balance test %d: Got %v Expected %v", i, bal, tst.bal) + } + } + if t.Failed() { + t.Fatal("Failed balance checks after spending coinbase with unmined transaction") + } + + // Mine the spending transaction in the block the coinbase matures. + bMaturity := BlockMeta{ + Block: Block{Height: b100.Height + blockchain.CoinbaseMaturity}, + Time: time.Now(), + } + err = s.InsertTx(spenderARec, &bMaturity) + if err != nil { + t.Fatal(err) + } + + balTests = []balTest{ + // Maturity height + { + height: bMaturity.Height, + minConf: 0, + bal: 35e8, + }, + { + height: bMaturity.Height, + minConf: 1, + bal: 35e8, + }, + { + height: bMaturity.Height, + minConf: 2, + bal: 30e8, + }, + { + height: bMaturity.Height, + minConf: blockchain.CoinbaseMaturity, + bal: 30e8, + }, + { + height: bMaturity.Height, + minConf: blockchain.CoinbaseMaturity + 1, + bal: 30e8, + }, + { + height: bMaturity.Height, + minConf: blockchain.CoinbaseMaturity + 2, + bal: 0, + }, + + // Next block after maturity height + { + height: bMaturity.Height + 1, + minConf: 0, + bal: 35e8, + }, + { + height: bMaturity.Height + 1, + minConf: 2, + bal: 35e8, + }, + { + height: bMaturity.Height + 1, + minConf: 3, + bal: 30e8, + }, + { + height: bMaturity.Height + 1, + minConf: blockchain.CoinbaseMaturity + 2, + bal: 30e8, + }, + { + height: bMaturity.Height + 1, + minConf: blockchain.CoinbaseMaturity + 3, + bal: 0, + }, + } + for i, tst := range balTests { + bal, err := s.Balance(tst.minConf, tst.height) + if err != nil { + t.Fatalf("Balance test %d: Store.Balance failed: %v", i, err) + } + if bal != tst.bal { + t.Errorf("Balance test %d: Got %v Expected %v", i, bal, tst.bal) + } + } + if t.Failed() { + t.Fatal("Failed balance checks mining coinbase spending transaction") + } + + // Create another spending transaction which spends the credit from the + // first spender. This will be used to test removing the entire + // conflict chain when the coinbase is later reorged out. + // + // Use the same output amount as spender A and mark it as a credit. + // This will mean the balance tests should report identical results. + spenderBTime := time.Now() + spenderB := spendOutput(&spenderARec.Hash, 0, 5e8) + spenderBRec, err := NewTxRecordFromMsgTx(spenderB, spenderBTime) + if err != nil { + t.Fatal(err) + } + err = s.InsertTx(spenderBRec, &bMaturity) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(spenderBRec, &bMaturity, 0, false) + if err != nil { + t.Fatal(err) + } + for i, tst := range balTests { + bal, err := s.Balance(tst.minConf, tst.height) + if err != nil { + t.Fatalf("Balance test %d: Store.Balance failed: %v", i, err) + } + if bal != tst.bal { + t.Errorf("Balance test %d: Got %v Expected %v", i, bal, tst.bal) + } + } + if t.Failed() { + t.Fatal("Failed balance checks mining second spending transaction") + } + + // Reorg out the block that matured the coinbase and check balances + // again. + err = s.Rollback(bMaturity.Height) + if err != nil { + t.Fatal(err) + } + balTests = balTestsBeforeMaturity + for i, tst := range balTests { + bal, err := s.Balance(tst.minConf, tst.height) + if err != nil { + t.Fatalf("Balance test %d: Store.Balance failed: %v", i, err) + } + if bal != tst.bal { + t.Errorf("Balance test %d: Got %v Expected %v", i, bal, tst.bal) + } + } + if t.Failed() { + t.Fatal("Failed balance checks after reorging maturity block") + } + + // Reorg out the block which contained the coinbase. There should be no + // more transactions in the store (since the previous outputs referenced + // by the spending tx no longer exist), and the balance will always be + // zero. + err = s.Rollback(b100.Height) + if err != nil { + t.Fatal(err) + } + balTests = []balTest{ + // Current height + { + height: b100.Height - 1, + minConf: 0, + bal: 0, + }, + { + height: b100.Height - 1, + minConf: 1, + bal: 0, + }, + + // Next height + { + height: b100.Height, + minConf: 0, + bal: 0, + }, + { + height: b100.Height, + minConf: 1, + bal: 0, + }, + } + for i, tst := range balTests { + bal, err := s.Balance(tst.minConf, tst.height) + if err != nil { + t.Fatalf("Balance test %d: Store.Balance failed: %v", i, err) + } + if bal != tst.bal { + t.Errorf("Balance test %d: Got %v Expected %v", i, bal, tst.bal) + } + } + if t.Failed() { + t.Fatal("Failed balance checks after reorging coinbase block") + } + unminedTxs, err := s.UnminedTxs() + if err != nil { + t.Fatal(err) + } + if len(unminedTxs) != 0 { + t.Fatalf("Should have no unmined transactions after coinbase reorg, found %d", len(unminedTxs)) + } +} + +// Test moving multiple transactions from unmined buckets to the same block. +func TestMoveMultipleToSameBlock(t *testing.T) { + t.Parallel() + + s, teardown, err := testStore() + defer teardown() + if err != nil { + t.Fatal(err) + } + + b100 := BlockMeta{ + Block: Block{Height: 100}, + Time: time.Now(), + } + + cb := newCoinBase(20e8, 30e8) + cbRec, err := NewTxRecordFromMsgTx(cb, b100.Time) + if err != nil { + t.Fatal(err) + } + + // Insert coinbase and mark both outputs as credits. + err = s.InsertTx(cbRec, &b100) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(cbRec, &b100, 0, false) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(cbRec, &b100, 1, false) + if err != nil { + t.Fatal(err) + } + + // Create and insert two unmined transactions which spend both coinbase + // outputs. + spenderATime := time.Now() + spenderA := spendOutput(&cbRec.Hash, 0, 1e8, 2e8, 18e8) + spenderARec, err := NewTxRecordFromMsgTx(spenderA, spenderATime) + if err != nil { + t.Fatal(err) + } + err = s.InsertTx(spenderARec, nil) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(spenderARec, nil, 0, false) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(spenderARec, nil, 1, false) + if err != nil { + t.Fatal(err) + } + spenderBTime := time.Now() + spenderB := spendOutput(&cbRec.Hash, 1, 4e8, 8e8, 18e8) + spenderBRec, err := NewTxRecordFromMsgTx(spenderB, spenderBTime) + if err != nil { + t.Fatal(err) + } + err = s.InsertTx(spenderBRec, nil) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(spenderBRec, nil, 0, false) + if err != nil { + t.Fatal(err) + } + err = s.AddCredit(spenderBRec, nil, 1, false) + if err != nil { + t.Fatal(err) + } + + // Mine both transactions in the block that matures the coinbase. + bMaturity := BlockMeta{ + Block: Block{Height: b100.Height + blockchain.CoinbaseMaturity}, + Time: time.Now(), + } + err = s.InsertTx(spenderARec, &bMaturity) + if err != nil { + t.Fatal(err) + } + err = s.InsertTx(spenderBRec, &bMaturity) + if err != nil { + t.Fatal(err) + } + + // Check that both transactions can be queried at the maturity block. + detailsA, err := s.UniqueTxDetails(&spenderARec.Hash, &bMaturity.Block) + if err != nil { + t.Fatal(err) + } + if detailsA == nil { + t.Fatal("No details found for first spender") + } + detailsB, err := s.UniqueTxDetails(&spenderBRec.Hash, &bMaturity.Block) + if err != nil { + t.Fatal(err) + } + if detailsB == nil { + t.Fatal("No details found for second spender") + } + + // Verify that the balance was correctly updated on the block record + // append and that no unmined transactions remain. + balTests := []struct { + height int32 + minConf int32 + bal btcutil.Amount + }{ + // Maturity height + { + height: bMaturity.Height, + minConf: 0, + bal: 15e8, + }, + { + height: bMaturity.Height, + minConf: 1, + bal: 15e8, + }, + { + height: bMaturity.Height, + minConf: 2, + bal: 0, + }, + + // Next block after maturity height + { + height: bMaturity.Height + 1, + minConf: 0, + bal: 15e8, + }, + { + height: bMaturity.Height + 1, + minConf: 2, + bal: 15e8, + }, + { + height: bMaturity.Height + 1, + minConf: 3, + bal: 0, + }, + } + for i, tst := range balTests { + bal, err := s.Balance(tst.minConf, tst.height) + if err != nil { + t.Fatalf("Balance test %d: Store.Balance failed: %v", i, err) + } + if bal != tst.bal { + t.Errorf("Balance test %d: Got %v Expected %v", i, bal, tst.bal) + } + } + if t.Failed() { + t.Fatal("Failed balance checks after moving both coinbase spenders") + } + unminedTxs, err := s.UnminedTxs() + if err != nil { + t.Fatal(err) + } + if len(unminedTxs) != 0 { + t.Fatalf("Should have no unmined transactions mining both, found %d", len(unminedTxs)) + } +} + +// Test the optional-ness of the serialized transaction in a TxRecord. +// NewTxRecord and NewTxRecordFromMsgTx both save the serialized transaction, so +// manually strip it out to test this code path. +func TestInsertUnserializedTx(t *testing.T) { + t.Parallel() + + s, teardown, err := testStore() + defer teardown() + if err != nil { + t.Fatal(err) + } + + tx := newCoinBase(50e8) + rec, err := NewTxRecordFromMsgTx(tx, timeNow()) + if err != nil { + t.Fatal(err) + } + b100 := makeBlockMeta(100) + err = s.InsertTx(stripSerializedTx(rec), &b100) + if err != nil { + t.Fatalf("Insert for stripped TxRecord failed: %v", err) + } + + // Ensure it can be retreived successfully. + details, err := s.UniqueTxDetails(&rec.Hash, &b100.Block) + if err != nil { + t.Fatal(err) + } + rec2, err := NewTxRecordFromMsgTx(&details.MsgTx, rec.Received) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(rec.SerializedTx, rec2.SerializedTx) { + t.Fatal("Serialized txs for coinbase do not match") + } + + // Now test that path with an unmined transaction. + tx = spendOutput(&rec.Hash, 0, 50e8) + rec, err = NewTxRecordFromMsgTx(tx, timeNow()) + if err != nil { + t.Fatal(err) + } + err = s.InsertTx(rec, nil) + if err != nil { + t.Fatal(err) + } + details, err = s.UniqueTxDetails(&rec.Hash, nil) + if err != nil { + t.Fatal(err) + } + rec2, err = NewTxRecordFromMsgTx(&details.MsgTx, rec.Received) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(rec.SerializedTx, rec2.SerializedTx) { + t.Fatal("Serialized txs for coinbase spender do not match") + } +} diff --git a/wtxmgr/unconfirmed.go b/wtxmgr/unconfirmed.go new file mode 100644 index 0000000..c14b3d2 --- /dev/null +++ b/wtxmgr/unconfirmed.go @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2013-2015 Conformal Systems LLC + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package wtxmgr + +import ( + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcwallet/walletdb" +) + +// insertMemPoolTx inserts the unmined transaction record. It also marks +// previous outputs referenced by the inputs as spent. +func (s *Store) insertMemPoolTx(ns walletdb.Bucket, rec *TxRecord) error { + v := existsRawUnmined(ns, rec.Hash[:]) + if v != nil { + // TODO: compare serialized txs to ensure this isn't a hash collision? + return nil + } + + log.Infof("Inserting unconfirmed transaction %v", rec.Hash) + v, err := valueTxRecord(rec) + if err != nil { + return err + } + err = putRawUnmined(ns, rec.Hash[:], v) + if err != nil { + return err + } + + for _, input := range rec.MsgTx.TxIn { + prevOut := &input.PreviousOutPoint + k := canonicalOutPoint(&prevOut.Hash, prevOut.Index) + err = putRawUnminedInput(ns, k, rec.Hash[:]) + if err != nil { + return err + } + } + + // TODO: increment credit amount for each credit (but those are unknown + // here currently). + + return nil +} + +// removeDoubleSpends checks for any unmined transactions which would introduce +// a double spend if tx was added to the store (either as a confirmed or unmined +// transaction). Each conflicting transaction and all transactions which spend +// it are recursively removed. +func (s *Store) removeDoubleSpends(ns walletdb.Bucket, rec *TxRecord) error { + for _, input := range rec.MsgTx.TxIn { + prevOut := &input.PreviousOutPoint + prevOutKey := canonicalOutPoint(&prevOut.Hash, prevOut.Index) + doubleSpendHash := existsRawUnminedInput(ns, prevOutKey) + if doubleSpendHash != nil { + var doubleSpend TxRecord + doubleSpendVal := existsRawUnmined(ns, doubleSpendHash) + copy(doubleSpend.Hash[:], doubleSpendHash) // Silly but need an array + err := readRawTxRecord(&doubleSpend.Hash, doubleSpendVal, + &doubleSpend) + if err != nil { + return err + } + + log.Debugf("Removing double spending transaction %v", + doubleSpend.Hash) + err = s.removeConflict(ns, &doubleSpend) + if err != nil { + return err + } + } + } + return nil +} + +// removeConflict removes an unmined transaction record and all spend chains +// deriving from it from the store. This is designed to remove transactions +// that would otherwise result in double spend conflicts if left in the store, +// and to remove transactions that spend coinbase transactions on reorgs. +func (s *Store) removeConflict(ns walletdb.Bucket, rec *TxRecord) error { + // For each potential credit for this record, each spender (if any) must + // be recursively removed as well. Once the spenders are removed, the + // credit is deleted. + numOuts := uint32(len(rec.MsgTx.TxOut)) + for i := uint32(0); i < numOuts; i++ { + k := canonicalOutPoint(&rec.Hash, i) + spenderHash := existsRawUnminedInput(ns, k) + if spenderHash != nil { + var spender TxRecord + spenderVal := existsRawUnmined(ns, spenderHash) + copy(spender.Hash[:], spenderHash) // Silly but need an array + err := readRawTxRecord(&spender.Hash, spenderVal, &spender) + if err != nil { + return err + } + + log.Debugf("Transaction %v is part of a removed conflict "+ + "chain -- removing as well", spender.Hash) + err = s.removeConflict(ns, &spender) + if err != nil { + return err + } + } + err := deleteRawUnminedCredit(ns, k) + if err != nil { + return err + } + } + + // If this tx spends any previous credits (either mined or unmined), set + // each unspent. Mined transactions are only marked spent by having the + // output in the unmined inputs bucket. + for _, input := range rec.MsgTx.TxIn { + prevOut := &input.PreviousOutPoint + k := canonicalOutPoint(&prevOut.Hash, prevOut.Index) + err := deleteRawUnminedInput(ns, k) + if err != nil { + return err + } + } + + return deleteRawUnmined(ns, rec.Hash[:]) +} + +// UnminedTxs returns the underlying transactions for all unmined transactions +// which are not known to have been mined in a block. +func (s *Store) UnminedTxs() ([]*wire.MsgTx, error) { + var txs []*wire.MsgTx + err := scopedView(s.namespace, func(ns walletdb.Bucket) error { + var err error + txs, err = s.unminedTxs(ns) + return err + }) + return txs, err +} + +func (s *Store) unminedTxs(ns walletdb.Bucket) ([]*wire.MsgTx, error) { + var unmined []*wire.MsgTx + err := ns.Bucket(bucketUnmined).ForEach(func(k, v []byte) error { + // TODO: Parsing transactions from the db may be a little + // expensive. It's possible the caller only wants the + // serialized transactions. + var txHash wire.ShaHash + err := readRawUnminedHash(k, &txHash) + if err != nil { + return err + } + + var rec TxRecord + err = readRawTxRecord(&txHash, v, &rec) + if err != nil { + return err + } + + tx := rec.MsgTx + unmined = append(unmined, &tx) + return nil + }) + return unmined, err +}