LBRYCommunity/lbcd
1
0
Fork 0
Code Issues 17 Pull requests 4 Projects Releases 28 Packages Wiki Activity

Prune the btcddb.Db interface.

Browse source 
This commit prunes several unused functions from the Db interface and the
underlying implementations.  For the most part these are holdovers from
the original sqlite implementation.  It also removes the types associated
with those functions since they are no longer needed.  The following
functions and types have been removed:

- InvalidateCache
- InvalidateBlockCache
- InvalidateTxCache
- SetDBInsertMode
  - InsertMode type and associated constants
- NewIterateBlocks
  - BlockIterator interface

The reasons for removing these are broken out below.

- Neither of two current implementations implement these functions nor
  does any of the fully functional code using the interface invoke them.
- After contemplating and testing caching of blocks and transactions at
  this layer, it doesn't seem to provide any real benefit unless very
  specific assumptions about the use case are made.  Making those
  assumptions can make other use cases worse.  For example, assuming a
  large cache is harmful to memory-constrained use cases.  Leaving it up
  to the caller to choose when to cache block and transactions allows much
  greater flexibility.
- The DB insert mode was an artifact of the original sqlite implementation
  and probably should have only been exposed specifically on the
  implementation as opposed through generic interface.  If a specific
  implementation wishes to provide functionality such as special modes,
  that should be done through type assertions.
This commit is contained in:
Dave Collins 2014-01-19 17:34:54 -06:00
parent 8c34084e24
commit 2eea55ae1d
9 changed files with 69 additions and 255 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
db.go
View file

@ -23,20 +23,6 @@ var (
// a range of shas by height to request them all.
const AllShas = int64(^uint64(0) >> 1)
// InsertMode represents a hint to the database about how much data the
// application is expecting to send to the database in a short period of time.
// This in turn provides the database with the opportunity to work in optimized
// modes when it will be very busy such as during the initial block chain
// download.
type InsertMode int
// Constants used to indicate the database insert mode hint. See InsertMode.
const (
InsertNormal InsertMode = iota
InsertFast
InsertValidatedInput
)
// Db defines a generic interface that is used to request and insert data into
// the bitcoin block chain. This interface is intended to be agnostic to actual
// mechanism used for backend data storage. The AddDBDriver function can be
@ -102,18 +88,6 @@ type Db interface {
// requires the referenced parent block to already exist.
InsertBlock(block *btcutil.Block) (height int64, err error)
// InvalidateBlockCache releases all cached blocks.
InvalidateBlockCache()
// InvalidateCache releases all cached blocks and transactions.
InvalidateCache()
// InvalidateTxCache releases all cached transactions.
InvalidateTxCache()
// NewIterateBlocks returns an iterator for all blocks in database.
NewIterateBlocks() (pbi BlockIterator, err error)
// NewestSha returns the hash and block height of the most recent (end)
// block of the block chain. It will return the zero hash, -1 for
// the block height, and no error (nil) if there are not any blocks in
@ -124,29 +98,11 @@ type Db interface {
// saved data at last Sync and closes the database.
RollbackClose()
// SetDBInsertMode provides hints to the database about how the
// application is running. This allows the database to work in
// optimized modes when the database may be very busy.
SetDBInsertMode(InsertMode)
// Sync verifies that the database is coherent on disk and no
// outstanding transactions are in flight.
Sync()
}
// BlockIterator defines a generic interface for an iterator through the block
// chain.
type BlockIterator interface {
// Close shuts down the iterator when done walking blocks in the database.
Close()
// NextRow iterates thru all blocks in database.
NextRow() bool
// Row returns row data for block iterator.
Row() (key *btcwire.ShaHash, pver uint32, buf []byte, err error)
}
// DriverDB defines a structure for backend drivers to use when they registered
// themselves as a backend which implements the Db interface.
type DriverDB struct {

91
interface_test.go
View file

@ -569,35 +569,23 @@ func testInterface(t *testing.T, dbType string) {
}
}
// The data integrity tests must still pass after calling each of the
// invalidate cache functions. This intentionally uses a map since
// map iteration is not the same order every run. This helps catch
// issues that could be caused by calling one version before another.
// Run the data integrity tests again after all blocks have been
// inserted to ensure the spend tracking is working properly.
context.useSpends = true
invalidateCacheFuncs := map[string]func(){
"InvalidateBlockCache": db.InvalidateBlockCache,
"InvalidateTxCache": db.InvalidateTxCache,
"InvalidateCache": db.InvalidateCache,
}
for funcName, invalidateCacheFunc := range invalidateCacheFuncs {
t.Logf("Running integrity tests after calling %s", funcName)
invalidateCacheFunc()
for height := int64(0); height < int64(len(blocks)); height++ {
// Get the appropriate block and hash and update the
// test context accordingly.
block := blocks[height]
blockHash, err := block.Sha()
if err != nil {
t.Errorf("block.Sha: %v", err)
return
}
context.blockHeight = height
context.blockHash = blockHash
context.block = block
testIntegrity(&context)
for height := int64(0); height < int64(len(blocks)); height++ {
// Get the appropriate block and hash and update the
// test context accordingly.
block := blocks[height]
blockHash, err := block.Sha()
if err != nil {
t.Errorf("block.Sha: %v", err)
return
}
context.blockHeight = height
context.blockHash = blockHash
context.block = block
testIntegrity(&context)
}
// TODO(davec): Need to figure out how to handle the special checks
@ -605,34 +593,31 @@ func testInterface(t *testing.T, dbType string) {
// 91880 on the main network due to the old miner + Satoshi client bug.
// TODO(davec): Add tests for error conditions:
// * Don't allow duplicate blocks
// * Don't allow insertion of block that contains a transaction that
// already exists unless the previous one is fully spent
// * Don't allow block that has a duplicate transaction in itself
// * Don't allow block which contains a tx that references a missing tx
// * Don't allow block which contains a tx that references another tx
// that comes after it in the same block
/*
- Don't allow duplicate blocks
- Don't allow insertion of block that contains a transaction that
already exists unless the previous one is fully spent
- Don't allow block that has a duplicate transaction in itself
- Don't allow block which contains a tx that references a missing tx
- Don't allow block which contains a tx that references another tx
that comes after it in the same block
*/
// TODO(davec): Add tests for the following functions:
/*
Close()
DropAfterBlockBySha(*btcwire.ShaHash) (err error)
- ExistsSha(sha *btcwire.ShaHash) (exists bool)
- FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error)
- FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error)
FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error)
- ExistsTxSha(sha *btcwire.ShaHash) (exists bool)
- FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error)
- FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply
- FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply
- InsertBlock(block *btcutil.Block) (height int64, err error)
- InvalidateBlockCache()
- InvalidateCache()
- InvalidateTxCache()
NewIterateBlocks() (pbi BlockIterator, err error)
NewestSha() (sha *btcwire.ShaHash, height int64, err error)
RollbackClose()
SetDBInsertMode(InsertMode)
Sync()
- Close()
- DropAfterBlockBySha(*btcwire.ShaHash) (err error)
x ExistsSha(sha *btcwire.ShaHash) (exists bool)
x FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error)
x FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error)
- FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error)
x ExistsTxSha(sha *btcwire.ShaHash) (exists bool)
x FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error)
x FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply
x FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply
x InsertBlock(block *btcutil.Block) (height int64, err error)
x NewestSha() (sha *btcwire.ShaHash, height int64, err error)
- RollbackClose()
- Sync()
*/
}

5
ldb/block.go
View file

@ -308,8 +308,3 @@ func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error)
return &sha, db.lastBlkIdx, nil
}
func (db *LevelDb) NewIterateBlocks() (rbogus btcdb.BlockIterator, err error) {
err = fmt.Errorf("Not implemented")
return
}

21
ldb/dbcache.go
View file

@ -1,21 +0,0 @@
// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
package ldb
import (
//"fmt"
)
// InvalidateTxCache clear/release all cached transactions.
func (db *LevelDb) InvalidateTxCache() {
}
// InvalidateTxCache clear/release all cached blocks.
func (db *LevelDb) InvalidateBlockCache() {
}
// InvalidateCache clear/release all cached blocks and transactions.
func (db *LevelDb) InvalidateCache() {
}

21
ldb/insertremove_test.go
View file

@ -33,9 +33,7 @@ func loadblocks(t *testing.T) []*btcutil.Block {
}
func TestUnspentInsert(t *testing.T) {
testUnspentInsert(t, dbTmDefault)
//testUnspentInsert(t, dbTmNormal)
//testUnspentInsert(t, dbTmFast)
testUnspentInsert(t)
}
// insert every block in the test chain
@ -43,9 +41,9 @@ func TestUnspentInsert(t *testing.T) {
// block and verify that the the tx is spent/unspent
// new tx should be fully unspent, referenced tx should have
// the associated txout set to spent.
func testUnspentInsert(t *testing.T, mode int) {
func testUnspentInsert(t *testing.T) {
// Ignore db remove errors since it means we didn't have an old one.
dbname := fmt.Sprintf("tstdbuspnt1.%d", mode)
dbname := fmt.Sprintf("tstdbuspnt1")
dbnamever := dbname + ".ver"
_ = os.RemoveAll(dbname)
_ = os.RemoveAll(dbnamever)
@ -58,19 +56,6 @@ func testUnspentInsert(t *testing.T, mode int) {
defer os.RemoveAll(dbnamever)
defer db.Close()
switch mode {
case dbTmDefault: // default
// no setup
case dbTmNormal: // explicit normal
db.SetDBInsertMode(btcdb.InsertNormal)
case dbTmFast: // fast mode
case dbTmNoVerify: // validated block
t.Errorf("UnspentInsert test is not valid in NoVerify mode")
}
// Since we are dealing with small dataset, reduce cache size
blocks := loadblocks(t)
endtest:
for height := int64(0); height < int64(len(blocks)); height++ {

8
ldb/leveldb.go
View file

@ -420,14 +420,6 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error)
return newheight, nil
}
// SetDBInsertMode provides hints to the database to how the application
// is running this allows the database to work in optimized modes when the
// database may be very busy.
func (db *LevelDb) SetDBInsertMode(newmode btcdb.InsertMode) {
// special modes are not supported
}
// doSpend iterates all TxIn in a bitcoin transaction marking each associated
// TxOut as spent.
func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error {

106
ldb/operational_test.go
View file

@ -20,28 +20,18 @@ import (
var network = btcwire.MainNet
const (
dbTmDefault = iota
dbTmNormal
dbTmFast
dbTmNoVerify
)
func TestOperational(t *testing.T) {
testOperationalMode(t, dbTmDefault)
//testOperationalMode(t, dbTmNormal)
//testOperationalMode(t, dbTmFast)
//testOperationalMode(t, dbTmNoVerify)
testOperationalMode(t)
}
func testOperationalMode(t *testing.T, mode int) {
func testOperationalMode(t *testing.T) {
// simplified basic operation is:
// 1) fetch block from remote server
// 2) look up all txin (except coinbase in db)
// 3) insert block
// Ignore db remove errors since it means we didn't have an old one.
dbname := fmt.Sprintf("tstdbop1.%d", mode)
dbname := fmt.Sprintf("tstdbop1")
dbnamever := dbname + ".ver"
_ = os.RemoveAll(dbname)
_ = os.RemoveAll(dbnamever)
@ -54,22 +44,10 @@ func testOperationalMode(t *testing.T, mode int) {
defer os.RemoveAll(dbnamever)
defer db.Close()
switch mode {
case dbTmDefault: // default
// no setup
case dbTmNormal: // explicit normal
db.SetDBInsertMode(btcdb.InsertNormal)
case dbTmFast: // fast mode
case dbTmNoVerify: // validated block
db.SetDBInsertMode(btcdb.InsertValidatedInput)
}
testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2")
blocks, err := loadBlocks(t, testdatafile)
if err != nil {
t.Errorf("Unable to load blocks from test data for mode %v: %v",
mode, err)
t.Errorf("Unable to load blocks from test data: %v", err)
return
}
@ -77,36 +55,31 @@ func testOperationalMode(t *testing.T, mode int) {
out:
for height := int64(0); height < int64(len(blocks)); height++ {
block := blocks[height]
if mode != dbTmNoVerify {
// except for NoVerify which does not allow lookups check inputs
mblock := block.MsgBlock()
var txneededList []*btcwire.ShaHash
for _, tx := range mblock.Transactions {
for _, txin := range tx.TxIn {
if txin.PreviousOutpoint.Index == uint32(4294967295) {
continue
}
origintxsha := &txin.PreviousOutpoint.Hash
txneededList = append(txneededList, origintxsha)
mblock := block.MsgBlock()
var txneededList []*btcwire.ShaHash
for _, tx := range mblock.Transactions {
for _, txin := range tx.TxIn {
if txin.PreviousOutpoint.Index == uint32(4294967295) {
continue
}
origintxsha := &txin.PreviousOutpoint.Hash
txneededList = append(txneededList, origintxsha)
if !db.ExistsTxSha(origintxsha) {
t.Errorf("referenced tx not found %v ", origintxsha)
}
_, err = db.FetchTxBySha(origintxsha)
if err != nil {
t.Errorf("referenced tx not found %v err %v ", origintxsha, err)
}
if !db.ExistsTxSha(origintxsha) {
t.Errorf("referenced tx not found %v ", origintxsha)
}
_, err = db.FetchTxBySha(origintxsha)
if err != nil {
t.Errorf("referenced tx not found %v err %v ", origintxsha, err)
}
}
txlist := db.FetchUnSpentTxByShaList(txneededList)
for _, txe := range txlist {
if txe.Err != nil {
t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err)
break out
}
}
txlist := db.FetchUnSpentTxByShaList(txneededList)
for _, txe := range txlist {
if txe.Err != nil {
t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err)
break out
}
}
newheight, err := db.InsertBlock(block)
@ -136,34 +109,20 @@ out:
// now that db is populated, do some additional test
testFetchRangeHeight(t, db, blocks)
switch mode {
case dbTmDefault: // default
// no cleanup
case dbTmNormal: // explicit normal
// no cleanup
case dbTmFast: // fast mode
db.SetDBInsertMode(btcdb.InsertNormal)
case dbTmNoVerify: // validated block
db.SetDBInsertMode(btcdb.InsertNormal)
}
}
func TestBackout(t *testing.T) {
testBackout(t, dbTmDefault)
//testBackout(t, dbTmNormal)
//testBackout(t, dbTmFast)
testBackout(t)
}
func testBackout(t *testing.T, mode int) {
func testBackout(t *testing.T) {
// simplified basic operation is:
// 1) fetch block from remote server
// 2) look up all txin (except coinbase in db)
// 3) insert block
t.Logf("mode %v", mode)
// Ignore db remove errors since it means we didn't have an old one.
dbname := fmt.Sprintf("tstdbop2.%d", mode)
dbname := fmt.Sprintf("tstdbop2")
dbnamever := dbname + ".ver"
_ = os.RemoveAll(dbname)
_ = os.RemoveAll(dbnamever)
@ -176,15 +135,6 @@ func testBackout(t *testing.T, mode int) {
defer os.RemoveAll(dbnamever)
defer db.Close()
switch mode {
case dbTmDefault: // default
// no setup
case dbTmNormal: // explicit normal
db.SetDBInsertMode(btcdb.InsertNormal)
case dbTmFast: // fast mode
}
testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2")
blocks, err := loadBlocks(t, testdatafile)
if len(blocks) < 120 {

21
memdb/memdb.go
View file

@ -703,15 +703,6 @@ func (db *MemDb) InvalidateTxCache() {
}
}
// NewIterateBlocks returns an iterator for all blocks in database. This is
// part of the btcdb.Db interface implementation.
//
// This implmentation does not implement an iterator, so an error is returned
// if this function is called.
func (db *MemDb) NewIterateBlocks() (btcdb.BlockIterator, error) {
return nil, fmt.Errorf("Not implemented")
}
// NewestSha returns the hash and block height of the most recent (end) block of
// the block chain. It will return the zero hash, -1 for the block height, and
// no error (nil) if there are not any blocks in the database yet. This is part
@ -752,18 +743,6 @@ func (db *MemDb) RollbackClose() {
db.Close()
}
// SetDBInsertMode provides hints to the database about how the application is
// running. This allows the database to work in optimized modes when the
// database may be very busy. This is part of the btcdb.Db interface
// implementation.
//
// No special mode handling is performed for this implementation.
func (db *MemDb) SetDBInsertMode(newmode btcdb.InsertMode) {
if db.closed {
log.Warnf("SetDBInsertMode called after db close.")
}
}
// Sync verifies that the database is coherent on disk and no outstanding
// transactions are in flight. This is part of the btcdb.Db interface
// implementation.

7
memdb/memdb_test.go
View file

@ -100,13 +100,6 @@ func TestClosed(t *testing.T) {
// The following calls don't return errors from the interface to be able
// to detect a closed database, so just call them to ensure there are no
// panics.
db.InvalidateBlockCache()
db.InvalidateCache()
db.InvalidateTxCache()
db.NewIterateBlocks()
db.SetDBInsertMode(btcdb.InsertNormal)
db.SetDBInsertMode(btcdb.InsertFast)
db.SetDBInsertMode(btcdb.InsertValidatedInput)
db.Sync()
db.RollbackClose()
}