Merge pull request #672 from cfromknecht/walletdb-sequence

walletdb: add sequence methods from bbolt to the main bucket interface

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0
 	github.com/btcsuite/btcwallet/wallet/txrules v1.0.0
-	github.com/btcsuite/btcwallet/walletdb v1.0.0
+	github.com/btcsuite/btcwallet/walletdb v1.2.0
 	github.com/btcsuite/btcwallet/wtxmgr v1.0.0
 	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -89,6 +89,8 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=

goclean.sh

@@ -7,7 +7,7 @@
 
 set -ex
 
-test_targets=$(go list ./...)
+test_targets=$(go list -deps ./... | grep 'btcwallet')
 
 # Automatic checks
 test -z "$(go fmt $test_targets | tee /dev/stderr)"

walletdb/bdb/db.go

@@ -68,7 +68,7 @@ func (tx *transaction) ReadWriteBucket(key []byte) walletdb.ReadWriteBucket {
 }
 
 func (tx *transaction) CreateTopLevelBucket(key []byte) (walletdb.ReadWriteBucket, error) {
-	boltBucket, err := tx.boltTx.CreateBucket(key)
+	boltBucket, err := tx.boltTx.CreateBucketIfNotExists(key)
 	if err != nil {
 		return nil, convertErr(err)
 	}
@@ -231,6 +231,21 @@ func (b *bucket) Tx() walletdb.ReadWriteTx {
 	}
 }
 
+// NextSequence returns an autoincrementing integer for the bucket.
+func (b *bucket) NextSequence() (uint64, error) {
+	return (*bbolt.Bucket)(b).NextSequence()
+}
+
+// SetSequence updates the sequence number for the bucket.
+func (b *bucket) SetSequence(v uint64) error {
+	return (*bbolt.Bucket)(b).SetSequence(v)
+}
+
+// Sequence returns the current integer for the bucket without incrementing it.
+func (b *bucket) Sequence() uint64 {
+	return (*bbolt.Bucket)(b).Sequence()
+}
+
 // cursor represents a cursor over key/value pairs and nested buckets of a
 // bucket.
 //
@@ -327,6 +342,19 @@ func (db *db) Close() error {
 	return convertErr((*bbolt.DB)(db).Close())
 }
 
+// Batch is similar to the package-level Update method, but it will attempt to
+// optismitcally combine the invocation of several transaction functions into a
+// single db write transaction.
+//
+// This function is part of the walletdb.Db interface implementation.
+func (db *db) Batch(f func(tx walletdb.ReadWriteTx) error) error {
+	return (*bbolt.DB)(db).Batch(func(btx *bbolt.Tx) error {
+		interfaceTx := transaction{btx}
+
+		return f(&interfaceTx)
+	})
+}
+
 // filesExists reports whether the named file or directory exists.
 func fileExists(name string) bool {
 	if _, err := os.Stat(name); err != nil {

walletdb/bdb/driver_test.go

@@ -23,7 +23,7 @@ func TestCreateOpenFail(t *testing.T) {
 	// Ensure that attempting to open a database that doesn't exist returns
 	// the expected error.
 	wantErr := walletdb.ErrDbDoesNotExist
-	if _, err := walletdb.Open(dbType, "noexist.db"); err != wantErr {
+	if _, err := walletdb.Open(dbType, "noexist.db", true); err != wantErr {
 		t.Errorf("Open: did not receive expected error - got %v, "+
 			"want %v", err, wantErr)
 		return
@@ -32,7 +32,7 @@ func TestCreateOpenFail(t *testing.T) {
 	// Ensure that attempting to open a database with the wrong number of
 	// parameters returns the expected error.
 	wantErr = fmt.Errorf("invalid arguments to %s.Open -- expected "+
-		"database path", dbType)
+		"database path and no-freelist-sync option", dbType)
 	if _, err := walletdb.Open(dbType, 1, 2, 3); err.Error() != wantErr.Error() {
 		t.Errorf("Open: did not receive expected error - got %v, "+
 			"want %v", err, wantErr)
@@ -43,7 +43,7 @@ func TestCreateOpenFail(t *testing.T) {
 	// the first parameter returns the expected error.
 	wantErr = fmt.Errorf("first argument to %s.Open is invalid -- "+
 		"expected database path string", dbType)
-	if _, err := walletdb.Open(dbType, 1); err.Error() != wantErr.Error() {
+	if _, err := walletdb.Open(dbType, 1, true); err.Error() != wantErr.Error() {
 		t.Errorf("Open: did not receive expected error - got %v, "+
 			"want %v", err, wantErr)
 		return
@@ -52,7 +52,7 @@ func TestCreateOpenFail(t *testing.T) {
 	// Ensure that attempting to create a database with the wrong number of
 	// parameters returns the expected error.
 	wantErr = fmt.Errorf("invalid arguments to %s.Create -- expected "+
-		"database path", dbType)
+		"database path and no-freelist-sync option", dbType)
 	if _, err := walletdb.Create(dbType, 1, 2, 3); err.Error() != wantErr.Error() {
 		t.Errorf("Create: did not receive expected error - got %v, "+
 			"want %v", err, wantErr)
@@ -63,7 +63,7 @@ func TestCreateOpenFail(t *testing.T) {
 	// the first parameter returns the expected error.
 	wantErr = fmt.Errorf("first argument to %s.Create is invalid -- "+
 		"expected database path string", dbType)
-	if _, err := walletdb.Create(dbType, 1); err.Error() != wantErr.Error() {
+	if _, err := walletdb.Create(dbType, 1, true); err.Error() != wantErr.Error() {
 		t.Errorf("Create: did not receive expected error - got %v, "+
 			"want %v", err, wantErr)
 		return
@@ -72,7 +72,7 @@ func TestCreateOpenFail(t *testing.T) {
 	// Ensure operations against a closed database return the expected
 	// error.
 	dbPath := "createfail.db"
-	db, err := walletdb.Create(dbType, dbPath)
+	db, err := walletdb.Create(dbType, dbPath, true)
 	if err != nil {
 		t.Errorf("Create: unexpected error: %v", err)
 		return
@@ -93,7 +93,7 @@ func TestCreateOpenFail(t *testing.T) {
 func TestPersistence(t *testing.T) {
 	// Create a new database to run tests against.
 	dbPath := "persistencetest.db"
-	db, err := walletdb.Create(dbType, dbPath)
+	db, err := walletdb.Create(dbType, dbPath, true)
 	if err != nil {
 		t.Errorf("Failed to create test database (%s) %v", dbType, err)
 		return

walletdb/db_test.go

@@ -55,7 +55,7 @@ func TestAddDuplicateDriver(t *testing.T) {
 	}
 
 	dbPath := "dupdrivertest.db"
-	db, err := walletdb.Create(dbType, dbPath)
+	db, err := walletdb.Create(dbType, dbPath, true)
 	if err != nil {
 		t.Errorf("failed to create database: %v", err)
 		return

walletdb/example_test.go

@@ -28,7 +28,7 @@ func ExampleCreate() {
 	// this, but it's done here in the example to ensure the example cleans
 	// up after itself.
 	dbPath := filepath.Join(os.TempDir(), "examplecreate.db")
-	db, err := walletdb.Create("bdb", dbPath)
+	db, err := walletdb.Create("bdb", dbPath, true)
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -47,7 +47,7 @@ var exampleNum = 0
 func exampleLoadDB() (walletdb.DB, func(), error) {
 	dbName := fmt.Sprintf("exampleload%d.db", exampleNum)
 	dbPath := filepath.Join(os.TempDir(), dbName)
-	db, err := walletdb.Create("bdb", dbPath)
+	db, err := walletdb.Create("bdb", dbPath, true)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -111,7 +111,7 @@ func Example_basicUsage() {
 	// this, but it's done here in the example to ensure the example cleans
 	// up after itself.
 	dbPath := filepath.Join(os.TempDir(), "exampleusage.db")
-	db, err := walletdb.Create("bdb", dbPath)
+	db, err := walletdb.Create("bdb", dbPath, true)
 	if err != nil {
 		fmt.Println(err)
 		return

walletdb/go.mod

@@ -7,5 +7,6 @@ require (
 	github.com/coreos/bbolt v1.3.3
 	github.com/davecgh/go-spew v1.1.1
 	go.etcd.io/bbolt v1.3.3 // indirect
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect
 )

walletdb/go.sum

@@ -6,5 +6,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

walletdb/interface.go

@@ -7,7 +7,10 @@
 
 package walletdb
 
-import "io"
+import (
+	"fmt"
+	"io"
+)
 
 // ReadTx represents a database transaction that can only be used for reads.  If
 // a database update must occur, use a ReadWriteTx.
@@ -126,6 +129,16 @@ type ReadWriteBucket interface {
 
 	// Tx returns the bucket's transaction.
 	Tx() ReadWriteTx
+
+	// NextSequence returns an autoincrementing integer for the bucket.
+	NextSequence() (uint64, error)
+
+	// SetSequence updates the sequence number for the bucket.
+	SetSequence(v uint64) error
+
+	// Sequence returns the current integer for the bucket without
+	// incrementing it.
+	Sequence() uint64
 }
 
 // ReadCursor represents a bucket cursor that can be positioned at the start or
@@ -191,6 +204,18 @@ type DB interface {
 	Close() error
 }
 
+// BatchDB is a special version of the main DB interface that allos the caller
+// to specify write transactions that should be combine dtoegether if multiple
+// goroutines are calling the Batch method.
+type BatchDB interface {
+	DB
+
+	// Batch is similar to the package-level Update method, but it will
+	// attempt to optismitcally combine the invocation of several
+	// transaction functions into a single db write transaction.
+	Batch(func(tx ReadWriteTx) error) error
+}
+
 // View opens a database read transaction and executes the function f with the
 // transaction passed as a parameter.  After f exits, the transaction is rolled
 // back.  If f errors, its error is returned, not a rollback error (if any
@@ -200,11 +225,20 @@ func View(db DB, f func(tx ReadTx) error) error {
 	if err != nil {
 		return err
 	}
+
+	// Make sure the transaction rolls back in the event of a panic.
+	defer func() {
+		if tx != nil {
+			tx.Rollback()
+		}
+	}()
+
 	err = f(tx)
 	rollbackErr := tx.Rollback()
 	if err != nil {
 		return err
 	}
+
 	if rollbackErr != nil {
 		return rollbackErr
 	}
@@ -222,6 +256,14 @@ func Update(db DB, f func(tx ReadWriteTx) error) error {
 	if err != nil {
 		return err
 	}
+
+	// Make sure the transaction rolls back in the event of a panic.
+	defer func() {
+		if tx != nil {
+			tx.Rollback()
+		}
+	}()
+
 	err = f(tx)
 	if err != nil {
 		// Want to return the original error, not a rollback error if
@@ -229,9 +271,27 @@ func Update(db DB, f func(tx ReadWriteTx) error) error {
 		_ = tx.Rollback()
 		return err
 	}
+
 	return tx.Commit()
 }
 
+// Batch opens a database read/write transaction and executes the function f
+// with the transaction passed as a parameter.  After f exits, if f did not
+// error, the transaction is committed.  Otherwise, if f did error, the
+// transaction is rolled back.  If the rollback fails, the original error
+// returned by f is still returned.  If the commit fails, the commit error is
+// returned.
+//
+// Batch is only useful when there are multiple goroutines calling it.
+func Batch(db DB, f func(tx ReadWriteTx) error) error {
+	batchDB, ok := db.(BatchDB)
+	if !ok {
+		return fmt.Errorf("need batch")
+	}
+
+	return batchDB.Batch(f)
+}
+
 // Driver defines a structure for backend drivers to use when they registered
 // themselves as a backend which implements the Db interface.
 type Driver struct {

walletdb/walletdbtest/interface.go

@@ -5,9 +5,11 @@
 package walletdbtest
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"reflect"
+	"sync"
 
 	"github.com/btcsuite/btcwallet/walletdb"
 )
@@ -104,6 +106,51 @@ func testNestedReadWriteBucket(tc *testContext, testBucket walletdb.ReadWriteBuc
 	return true
 }
 
+// testSequence tests that the sequence related methods work as expected.
+func testSequence(tc *testContext, testBucket walletdb.ReadWriteBucket) bool {
+	// Obtaining the current sequence twice should give us the same value.
+	seqNo1 := testBucket.Sequence()
+	seqNo2 := testBucket.Sequence()
+	if seqNo1 != seqNo2 {
+		tc.t.Errorf("Sequence: seq has incremented")
+		return false
+	}
+
+	// Incrementing to the next sequence should give us a value one larger
+	// than the prior number.
+	seqNo3, err := testBucket.NextSequence()
+	if err != nil {
+		tc.t.Errorf("Sequence: unexpected error: %v", err)
+		return false
+	}
+	if seqNo3 != seqNo2+1 {
+		tc.t.Errorf("Sequence: expected seq no of %v, instead got %v",
+			seqNo2+1, seqNo3)
+		return false
+	}
+
+	// We should be able to modify the sequence base number.
+	newBase := uint64(100)
+	if err := testBucket.SetSequence(newBase); err != nil {
+		tc.t.Errorf("Sequence: unexpected error: %v", err)
+		return false
+	}
+
+	// Any offset from this new sequence should now be properly reflected.
+	seqNo4, err := testBucket.NextSequence()
+	if err != nil {
+		tc.t.Errorf("Sequence: unexpected error: %v", err)
+		return false
+	}
+	if seqNo4 != newBase+1 {
+		tc.t.Errorf("Sequence: expected seq no of %v, instead got %v",
+			newBase+1, seqNo4)
+		return false
+	}
+
+	return true
+}
+
 // testReadWriteBucketInterface ensures the bucket interface is working properly by
 // exercising all of its functions.
 func testReadWriteBucketInterface(tc *testContext, bucket walletdb.ReadWriteBucket) bool {
@@ -164,6 +211,11 @@ func testReadWriteBucketInterface(tc *testContext, bucket walletdb.ReadWriteBuck
 		return false
 	}
 
+	// Test that the sequence methods work as expected.
+	if !testSequence(tc, bucket) {
+		return false
+	}
+
 	// Ensure creating a new bucket works as expected.
 	testBucketName := []byte("testbucket")
 	testBucket, err := bucket.CreateBucket(testBucketName)
@@ -676,9 +728,73 @@ func testAdditionalErrors(tc *testContext) bool {
 	return true
 }
 
+// testBatchInterface tests that if the target database implements the batch
+// method, then the method functions as expected.
+func testBatchInterface(tc *testContext) bool {
+	// If the database doesn't support the batch super-set of the
+	// interface, then we're done here.
+	batchDB, ok := tc.db.(walletdb.BatchDB)
+	if !ok {
+		return true
+	}
+
+	const numGoroutines = 5
+	errChan := make(chan error, numGoroutines)
+
+	var wg sync.WaitGroup
+	for i := 0; i < numGoroutines; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			err := walletdb.Batch(batchDB, func(tx walletdb.ReadWriteTx) error {
+				b, err := tx.CreateTopLevelBucket([]byte("test"))
+				if err != nil {
+					return err
+				}
+
+				byteI := []byte{byte(i)}
+				return b.Put(byteI, byteI)
+			})
+			errChan <- err
+		}(i)
+	}
+
+	wg.Wait()
+	close(errChan)
+
+	for err := range errChan {
+		if err != nil {
+			tc.t.Errorf("Batch: unexpected error: %v", err)
+			return false
+		}
+	}
+
+	err := walletdb.View(batchDB, func(tx walletdb.ReadTx) error {
+		b := tx.ReadBucket([]byte("test"))
+
+		for i := 0; i < numGoroutines; i++ {
+			byteI := []byte{byte(i)}
+			if v := b.Get(byteI); v == nil {
+				return fmt.Errorf("key %v not present", byteI)
+			} else if !bytes.Equal(v, byteI) {
+				return fmt.Errorf("key %v not equal to value: "+
+					"%v", byteI, v)
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		tc.t.Errorf("Batch: unexpected error: %v", err)
+		return false
+	}
+
+	return true
+}
+
 // TestInterface performs all interfaces tests for this database driver.
 func TestInterface(t Tester, dbType, dbPath string) {
-	db, err := walletdb.Create(dbType, dbPath)
+	db, err := walletdb.Create(dbType, dbPath, true)
 	if err != nil {
 		t.Errorf("Failed to create test database (%s) %v", dbType, err)
 		return
@@ -704,4 +820,9 @@ func TestInterface(t Tester, dbType, dbPath string) {
 	if !testAdditionalErrors(&context) {
 		return
 	}
+
+	// If applicable, also test the behavior of the Batch call.
+	if !testBatchInterface(&context) {
+		return
+	}
 }

wtxmgr/go.mod

@@ -6,5 +6,5 @@ require (
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
-	github.com/btcsuite/btcwallet/walletdb v1.0.0
+	github.com/btcsuite/btcwallet/walletdb v1.2.0
 )

wtxmgr/go.sum

@@ -5,8 +5,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcwallet/walletdb v1.0.0 h1:mheT7vCWK5EP6rZzhxsQ7ms9+yX4VE8bwiJctECBeNw=
+github.com/btcsuite/btcwallet/walletdb v1.1.0 h1:JHAL7wZ8pX4SULabeAv/wPO9sseRWMGzE80lfVmRw6Y=
-github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
+github.com/btcsuite/btcwallet/walletdb v1.1.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
 github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
 github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=

wtxmgr/tx_test.go

@@ -51,7 +51,7 @@ func testDB() (walletdb.DB, func(), error) {
 	if err != nil {
 		return nil, func() {}, err
 	}
-	db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"))
+	db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), true)
 	return db, func() { os.RemoveAll(tmpDir) }, err
 }
 
@@ -63,7 +63,7 @@ func testStore() (*Store, walletdb.DB, func(), error) {
 		return nil, nil, func() {}, err
 	}
 
-	db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"))
+	db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), true)
 	if err != nil {
 		os.RemoveAll(tmpDir)
 		return nil, nil, nil, err