lbcd/database2/ffldb/driver_test.go
Dave Collins af3ed803f5 database: Major redesign of database package.
This commit contains a complete redesign and rewrite of the database
package that approaches things in a vastly different manner than the
previous version.  This is the first part of several stages that will be
needed to ultimately make use of this new package.

Some of the reason for this were discussed in #255, however a quick
summary is as follows:

- The previous database could only contain blocks on the main chain and
  reorgs required deleting the blocks from the database.  This made it
  impossible to store orphans and could make external RPC calls for
  information about blocks during the middle of a reorg fail.
- The previous database interface forced a high level of bitcoin-specific
  intelligence such as spend tracking into each backend driver.
- The aforementioned point led to making it difficult to implement new
  backend drivers due to the need to repeat a lot of non-trivial logic
  which is better handled at a higher layer, such as the blockchain
  package.
- The old database stored all blocks in leveldb.  This made it extremely
  inefficient to do things such as lookup headers and individual
  transactions since the entire block had to be loaded from leveldb (which
  entails it doing data copies) to get access.

In order to address all of these concerns, and others not mentioned, the
database interface has been redesigned as follows:

- Two main categories of functionality are provided: block storage and
  metadata storage
- All block storage and metadata storage are done via read-only and
  read-write MVCC transactions with both manual and managed modes
  - Support for multiple concurrent readers and a single writer
  - Readers use a snapshot and therefore are not blocked by the writer
- Some key properties of the block storage and retrieval API:
  - It is generic and does NOT contain additional bitcoin logic such spend
    tracking and block linking
  - Provides access to the raw serialized bytes so deserialization is not
    forced for callers that don't need it
  - Support for fetching headers via independent functions which allows
    implementations to provide significant optimizations
  - Ability to efficiently retrieve arbitrary regions of blocks
    (transactions, scripts, etc)
- A rich metadata storage API is provided:
  - Key/value with arbitrary data
  - Support for buckets and nested buckets
  - Bucket iteration through a couple of different mechanisms
  - Cursors for efficient and direct key seeking
- Supports registration of backend database implementations
- Comprehensive test coverage
- Provides strong documentation with example usage

This commit also contains an implementation of the previously discussed
interface named ffldb (flat file plus leveldb metadata backend).  Here
is a quick overview:

- Highly optimized for read performance with consistent write performance
  regardless of database size
- All blocks are stored in flat files on the file system
- Bulk block region fetching is optimized to perform linear reads which
  improves performance on spindle disks
- Anti-corruption mechanisms:
  - Flat files contain full block checksums to quickly an easily detect
    database corruption without needing to do expensive merkle root
    calculations
  - Metadata checksums
  - Open reconciliation
- Extensive test coverage:
  - Comprehensive blackbox interface testing
  - Whitebox testing which uses intimate knowledge to exercise uncommon
    failure paths such as deleting files out from under the database
  - Corruption tests (replacing random data in the files)

In addition, this commit also contains a new tool under the new database
directory named dbtool which provides a few basic commands for testing the
database.  It is designed around commands, so it could be useful to expand
on in the future.

Finally, this commit addresses the following issues:

- Adds support for and therefore closes #255
- Fixes #199
- Fixes #201
- Implements and closes #256
- Obsoletes and closes #257
- Closes #247 once the required chain and btcd modifications are in place
  to make use of this new code
2016-02-03 11:42:04 -06:00

288 lines
8.1 KiB
Go

// Copyright (c) 2015-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
package ffldb_test
import (
"fmt"
"os"
"path/filepath"
"reflect"
"runtime"
"testing"
"github.com/btcsuite/btcd/chaincfg"
database "github.com/btcsuite/btcd/database2"
"github.com/btcsuite/btcd/database2/ffldb"
"github.com/btcsuite/btcutil"
)
// dbType is the database type name for this driver.
const dbType = "ffldb"
// TestCreateOpenFail ensures that errors related to creating and opening a
// database are handled properly.
func TestCreateOpenFail(t *testing.T) {
t.Parallel()
// Ensure that attempting to open a database that doesn't exist returns
// the expected error.
wantErrCode := database.ErrDbDoesNotExist
_, err := database.Open(dbType, "noexist", blockDataNet)
if !checkDbError(t, "Open", err, wantErrCode) {
return
}
// Ensure that attempting to open a database with the wrong number of
// parameters returns the expected error.
wantErr := fmt.Errorf("invalid arguments to %s.Open -- expected "+
"database path and block network", dbType)
_, err = database.Open(dbType, 1, 2, 3)
if err.Error() != wantErr.Error() {
t.Errorf("Open: did not receive expected error - got %v, "+
"want %v", err, wantErr)
return
}
// Ensure that attempting to open a database with an invalid type for
// the first parameter returns the expected error.
wantErr = fmt.Errorf("first argument to %s.Open is invalid -- "+
"expected database path string", dbType)
_, err = database.Open(dbType, 1, blockDataNet)
if err.Error() != wantErr.Error() {
t.Errorf("Open: did not receive expected error - got %v, "+
"want %v", err, wantErr)
return
}
// Ensure that attempting to open a database with an invalid type for
// the second parameter returns the expected error.
wantErr = fmt.Errorf("second argument to %s.Open is invalid -- "+
"expected block network", dbType)
_, err = database.Open(dbType, "noexist", "invalid")
if err.Error() != wantErr.Error() {
t.Errorf("Open: did not receive expected error - got %v, "+
"want %v", err, wantErr)
return
}
// Ensure that attempting to create a database with the wrong number of
// parameters returns the expected error.
wantErr = fmt.Errorf("invalid arguments to %s.Create -- expected "+
"database path and block network", dbType)
_, err = database.Create(dbType, 1, 2, 3)
if err.Error() != wantErr.Error() {
t.Errorf("Create: did not receive expected error - got %v, "+
"want %v", err, wantErr)
return
}
// Ensure that attempting to create a database with an invalid type for
// the first parameter returns the expected error.
wantErr = fmt.Errorf("first argument to %s.Create is invalid -- "+
"expected database path string", dbType)
_, err = database.Create(dbType, 1, blockDataNet)
if err.Error() != wantErr.Error() {
t.Errorf("Create: did not receive expected error - got %v, "+
"want %v", err, wantErr)
return
}
// Ensure that attempting to create a database with an invalid type for
// the second parameter returns the expected error.
wantErr = fmt.Errorf("second argument to %s.Create is invalid -- "+
"expected block network", dbType)
_, err = database.Create(dbType, "noexist", "invalid")
if err.Error() != wantErr.Error() {
t.Errorf("Create: did not receive expected error - got %v, "+
"want %v", err, wantErr)
return
}
// Ensure operations against a closed database return the expected
// error.
dbPath := filepath.Join(os.TempDir(), "ffldb-createfail")
_ = os.RemoveAll(dbPath)
db, err := database.Create(dbType, dbPath, blockDataNet)
if err != nil {
t.Errorf("Create: unexpected error: %v", err)
return
}
defer os.RemoveAll(dbPath)
db.Close()
wantErrCode = database.ErrDbNotOpen
err = db.View(func(tx database.Tx) error {
return nil
})
if !checkDbError(t, "View", err, wantErrCode) {
return
}
wantErrCode = database.ErrDbNotOpen
err = db.Update(func(tx database.Tx) error {
return nil
})
if !checkDbError(t, "Update", err, wantErrCode) {
return
}
wantErrCode = database.ErrDbNotOpen
_, err = db.Begin(false)
if !checkDbError(t, "Begin(false)", err, wantErrCode) {
return
}
wantErrCode = database.ErrDbNotOpen
_, err = db.Begin(true)
if !checkDbError(t, "Begin(true)", err, wantErrCode) {
return
}
wantErrCode = database.ErrDbNotOpen
err = db.Close()
if !checkDbError(t, "Close", err, wantErrCode) {
return
}
}
// TestPersistence ensures that values stored are still valid after closing and
// reopening the database.
func TestPersistence(t *testing.T) {
t.Parallel()
// Create a new database to run tests against.
dbPath := filepath.Join(os.TempDir(), "ffldb-persistencetest")
_ = os.RemoveAll(dbPath)
db, err := database.Create(dbType, dbPath, blockDataNet)
if err != nil {
t.Errorf("Failed to create test database (%s) %v", dbType, err)
return
}
defer os.RemoveAll(dbPath)
defer db.Close()
// Create a bucket, put some values into it, and store a block so they
// can be tested for existence on re-open.
bucket1Key := []byte("bucket1")
storeValues := map[string]string{
"b1key1": "foo1",
"b1key2": "foo2",
"b1key3": "foo3",
}
genesisBlock := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock)
genesisHash := chaincfg.MainNetParams.GenesisHash
err = db.Update(func(tx database.Tx) error {
metadataBucket := tx.Metadata()
if metadataBucket == nil {
return fmt.Errorf("Metadata: unexpected nil bucket")
}
bucket1, err := metadataBucket.CreateBucket(bucket1Key)
if err != nil {
return fmt.Errorf("CreateBucket: unexpected error: %v",
err)
}
for k, v := range storeValues {
err := bucket1.Put([]byte(k), []byte(v))
if err != nil {
return fmt.Errorf("Put: unexpected error: %v",
err)
}
}
if err := tx.StoreBlock(genesisBlock); err != nil {
return fmt.Errorf("StoreBlock: unexpected error: %v",
err)
}
return nil
})
if err != nil {
t.Errorf("Update: unexpected error: %v", err)
return
}
// Close and reopen the database to ensure the values persist.
db.Close()
db, err = database.Open(dbType, dbPath, blockDataNet)
if err != nil {
t.Errorf("Failed to open test database (%s) %v", dbType, err)
return
}
defer db.Close()
// Ensure the values previously stored in the 3rd namespace still exist
// and are correct.
err = db.View(func(tx database.Tx) error {
metadataBucket := tx.Metadata()
if metadataBucket == nil {
return fmt.Errorf("Metadata: unexpected nil bucket")
}
bucket1 := metadataBucket.Bucket(bucket1Key)
if bucket1 == nil {
return fmt.Errorf("Bucket1: unexpected nil bucket")
}
for k, v := range storeValues {
gotVal := bucket1.Get([]byte(k))
if !reflect.DeepEqual(gotVal, []byte(v)) {
return fmt.Errorf("Get: key '%s' does not "+
"match expected value - got %s, want %s",
k, gotVal, v)
}
}
genesisBlockBytes, _ := genesisBlock.Bytes()
gotBytes, err := tx.FetchBlock(genesisHash)
if err != nil {
return fmt.Errorf("FetchBlock: unexpected error: %v",
err)
}
if !reflect.DeepEqual(gotBytes, genesisBlockBytes) {
return fmt.Errorf("FetchBlock: stored block mismatch")
}
return nil
})
if err != nil {
t.Errorf("View: unexpected error: %v", err)
return
}
}
// TestInterface performs all interfaces tests for this database driver.
func TestInterface(t *testing.T) {
t.Parallel()
// Create a new database to run tests against.
dbPath := filepath.Join(os.TempDir(), "ffldb-interfacetest")
_ = os.RemoveAll(dbPath)
db, err := database.Create(dbType, dbPath, blockDataNet)
if err != nil {
t.Errorf("Failed to create test database (%s) %v", dbType, err)
return
}
defer os.RemoveAll(dbPath)
defer db.Close()
// Ensure the driver type is the expected value.
gotDbType := db.Type()
if gotDbType != dbType {
t.Errorf("Type: unepxected driver type - got %v, want %v",
gotDbType, dbType)
return
}
// Run all of the interface tests against the database.
runtime.GOMAXPROCS(runtime.NumCPU())
// Change the maximum file size to a small value to force multiple flat
// files with the test data set.
ffldb.TstRunWithMaxBlockFileSize(db, 2048, func() {
testInterface(t, db)
})
}