Initial leveldb code.
This code is still prototype at this time. It appears to function correctly but something consumes more memory than is considered reasonable for the dataset comprised of the full bitcoind chain. Not recommened for use at this time.
This commit is contained in:
Dale Rahn
Dale Rahn
parent
b4c6a5b8ab
commit
b66abdf6ba
10 changed files with 1792 additions and 0 deletions
262
ldb/block.go
Normal file
262
ldb/block.go
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"github.com/conformal/btcdb"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
// InsertBlockData stores a block hash and its associated data block with a
|
||||
// previous sha of `prevSha' and a version of `pver'.
|
||||
func (db *LevelDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
return db.insertBlockData(sha, prevSha, buf)
|
||||
}
|
||||
|
||||
func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) {
|
||||
var blkHeight int64
|
||||
|
||||
key := sha.Bytes()
|
||||
|
||||
data, err := db.bShaDb.Get(key, db.ro)
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// deserialize
|
||||
dr := bytes.NewBuffer(data)
|
||||
err = binary.Read(dr, binary.LittleEndian, &blkHeight)
|
||||
if err != nil {
|
||||
fmt.Printf("get getBlkLoc len %v\n", len(data))
|
||||
err = fmt.Errorf("Db Corrupt 0")
|
||||
return 0, err
|
||||
}
|
||||
return blkHeight, nil
|
||||
}
|
||||
|
||||
func (db *LevelDb) getBlkByHeight(blkHeight int64) (rsha *btcwire.ShaHash, rbuf []byte, err error) {
|
||||
var blkVal []byte
|
||||
|
||||
key := int64ToKey(blkHeight)
|
||||
|
||||
blkVal, err = db.bBlkDb.Get(key, db.ro)
|
||||
if err != nil {
|
||||
return // exists ???
|
||||
}
|
||||
|
||||
var sha btcwire.ShaHash
|
||||
|
||||
sha.SetBytes(blkVal[0:32])
|
||||
|
||||
blockdata := make([]byte, len(blkVal[32:]))
|
||||
copy(blockdata[:], blkVal[32:])
|
||||
|
||||
return &sha, blockdata, nil
|
||||
}
|
||||
|
||||
func (db *LevelDb) getBlk(sha *btcwire.ShaHash) (rblkHeight int64, rbuf []byte, err error) {
|
||||
var blkHeight int64
|
||||
|
||||
blkHeight, err = db.getBlkLoc(sha)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var buf []byte
|
||||
|
||||
_, buf, err = db.getBlkByHeight(blkHeight)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return blkHeight, buf, nil
|
||||
}
|
||||
|
||||
func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) error {
|
||||
|
||||
// serialize
|
||||
var lw bytes.Buffer
|
||||
err := binary.Write(&lw, binary.LittleEndian, blkHeight)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Write Fail")
|
||||
return err
|
||||
}
|
||||
shaKey := sha.Bytes()
|
||||
|
||||
blkKey := int64ToKey(blkHeight)
|
||||
|
||||
shaB := sha.Bytes()
|
||||
blkVal := make([]byte, len(shaB)+len(buf))
|
||||
copy(blkVal[0:], shaB)
|
||||
copy(blkVal[len(shaB):], buf)
|
||||
|
||||
db.bShaBatch().Put(shaKey, lw.Bytes())
|
||||
|
||||
db.bBlkBatch().Put(blkKey, blkVal)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// insertSha stores a block hash and its associated data block with a
|
||||
// previous sha of `prevSha' and a version of `pver'.
|
||||
// insertSha shall be called with db lock held
|
||||
func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, buf []byte) (blockid int64, err error) {
|
||||
|
||||
oBlkHeight, err := db.getBlkLoc(prevSha)
|
||||
|
||||
if err != nil {
|
||||
// check current block count
|
||||
// if count != 0 {
|
||||
// err = btcdb.PrevShaMissing
|
||||
// return
|
||||
// }
|
||||
oBlkHeight = -1
|
||||
}
|
||||
|
||||
// TODO(drahn) check curfile filesize, increment curfile if this puts it over
|
||||
blkHeight := oBlkHeight + 1
|
||||
|
||||
err = db.setBlk(sha, blkHeight, buf)
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// update the last block cache
|
||||
db.lastBlkShaCached = true
|
||||
db.lastBlkSha = *sha
|
||||
db.lastBlkIdx = blkHeight
|
||||
db.nextBlock = blkHeight + 1
|
||||
|
||||
return blkHeight, nil
|
||||
}
|
||||
|
||||
// fetchSha returns the datablock and pver for the given ShaHash.
|
||||
func (db *LevelDb) fetchSha(sha *btcwire.ShaHash) (rbuf []byte,
|
||||
rblkHeight int64, err error) {
|
||||
var blkHeight int64
|
||||
var buf []byte
|
||||
|
||||
blkHeight, buf, err = db.getBlk(sha)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return buf, blkHeight, nil
|
||||
}
|
||||
|
||||
// ExistsSha looks up the given block hash
|
||||
// returns true if it is present in the database.
|
||||
func (db *LevelDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
// not in cache, try database
|
||||
exists = db.blkExistsSha(sha)
|
||||
return
|
||||
}
|
||||
|
||||
// blkExistsSha looks up the given block hash
|
||||
// returns true if it is present in the database.
|
||||
// CALLED WITH LOCK HELD
|
||||
func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) bool {
|
||||
|
||||
_, err := db.getBlkLoc(sha)
|
||||
|
||||
if err != nil {
|
||||
/*
|
||||
should this warn if the failure is something besides does not exist ?
|
||||
log.Warnf("blkExistsSha: fail %v", err)
|
||||
*/
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// FetchBlockShaByHeight returns a block hash based on its height in the
|
||||
// block chain.
|
||||
func (db *LevelDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
return db.fetchBlockShaByHeight(height)
|
||||
}
|
||||
|
||||
// fetchBlockShaByHeight returns a block hash based on its height in the
|
||||
// block chain.
|
||||
func (db *LevelDb) fetchBlockShaByHeight(height int64) (rsha *btcwire.ShaHash, err error) {
|
||||
var sha *btcwire.ShaHash
|
||||
sha, _, err = db.getBlkByHeight(height)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return sha, nil
|
||||
}
|
||||
|
||||
// FetchHeightRange looks up a range of blocks by the start and ending
|
||||
// heights. Fetch is inclusive of the start height and exclusive of the
|
||||
// ending height. To fetch all hashes from the start height until no
|
||||
// more are present, use the special id `AllShas'.
|
||||
func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
var endidx int64
|
||||
if endHeight == btcdb.AllShas {
|
||||
endidx = startHeight + 500
|
||||
} else {
|
||||
endidx = endHeight
|
||||
}
|
||||
|
||||
var shalist []btcwire.ShaHash
|
||||
for height := startHeight; height < endidx; height++ {
|
||||
// TODO(drahn) fix blkFile from height
|
||||
|
||||
key := int64ToKey(height)
|
||||
blkVal, lerr := db.bBlkDb.Get(key, db.ro)
|
||||
if lerr != nil {
|
||||
break
|
||||
}
|
||||
|
||||
var sha btcwire.ShaHash
|
||||
sha.SetBytes(blkVal[0:32])
|
||||
shalist = append(shalist, sha)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
//log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startHeight, endHeight, len(shalist), err)
|
||||
|
||||
return shalist, nil
|
||||
}
|
||||
|
||||
// NewestSha returns the hash and block height of the most recent (end) block of
|
||||
// the block chain. It will return the zero hash, -1 for the block height, and
|
||||
// no error (nil) if there are not any blocks in the database yet.
|
||||
func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
if db.lastBlkIdx == -1 {
|
||||
err = fmt.Errorf("Empty Database")
|
||||
return
|
||||
}
|
||||
sha := db.lastBlkSha
|
||||
|
||||
return &sha, db.lastBlkIdx, nil
|
||||
}
|
||||
|
||||
func (db *LevelDb) NewIterateBlocks() (rbogus btcdb.BlockIterator, err error) {
|
||||
err = fmt.Errorf("Not implemented")
|
||||
return
|
||||
}
|
||||
136
ldb/dbcache.go
Normal file
136
ldb/dbcache.go
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/conformal/btcdb"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
// FetchBlockBySha - return a btcutil Block, object may be a cached.
|
||||
func (db *LevelDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
return db.fetchBlockBySha(sha)
|
||||
}
|
||||
|
||||
// fetchBlockBySha - return a btcutil Block, object may be a cached.
|
||||
// Must be called with db lock held.
|
||||
func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) {
|
||||
|
||||
buf, height, err := db.fetchSha(sha)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
blk, err = btcutil.NewBlockFromBytes(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
blk.SetHeight(height)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// FetchTxByShaList given a array of ShaHash, look up the transactions
|
||||
// and return them in a TxListReply array.
|
||||
func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
replies := make([]*btcdb.TxListReply, len(txShaList))
|
||||
for i, txsha := range txShaList {
|
||||
tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha)
|
||||
btxspent := []bool{}
|
||||
if err == nil {
|
||||
btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut))
|
||||
for idx := range tx.TxOut {
|
||||
byteidx := idx / 8
|
||||
byteoff := uint(idx % 8)
|
||||
btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0
|
||||
}
|
||||
}
|
||||
txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err}
|
||||
replies[i] = &txlre
|
||||
}
|
||||
return replies
|
||||
}
|
||||
|
||||
// fetchTxDataBySha returns several pieces of data regarding the given sha.
|
||||
func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) {
|
||||
var pver uint32
|
||||
var blksha *btcwire.ShaHash
|
||||
var blkHeight int64
|
||||
var txspent []byte
|
||||
var txOff, txLen int
|
||||
var blkbuf []byte
|
||||
|
||||
blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha)
|
||||
if err != nil {
|
||||
err = btcdb.TxShaMissing
|
||||
return
|
||||
}
|
||||
|
||||
blksha, blkbuf, err = db.getBlkByHeight(blkHeight)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to get block %v %v\n", blkHeight, err)
|
||||
return
|
||||
}
|
||||
|
||||
//log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n",
|
||||
// txsha, blksha, blkHeight, txOff, txLen)
|
||||
|
||||
txbuf := make([]byte, txLen)
|
||||
copy(txbuf[:], blkbuf[txOff:txOff+txLen])
|
||||
rbuf := bytes.NewBuffer(txbuf)
|
||||
|
||||
var tx btcwire.MsgTx
|
||||
err = tx.Deserialize(rbuf)
|
||||
if err != nil {
|
||||
log.Warnf("unable to decode tx block %v %v txoff %v txlen %v",
|
||||
blkHeight, blksha, txOff, txLen)
|
||||
return
|
||||
}
|
||||
|
||||
return &tx, txbuf, pver, blksha, blkHeight, txspent, nil
|
||||
}
|
||||
|
||||
// FetchTxAllBySha returns several pieces of data regarding the given sha.
|
||||
func (db *LevelDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
tx, txbuf, pver, blksha, _, _, err := db.fetchTxDataBySha(txsha)
|
||||
|
||||
return tx, txbuf, pver, blksha, nil
|
||||
}
|
||||
|
||||
// FetchTxBySha returns some data for the given Tx Sha.
|
||||
func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) {
|
||||
rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha)
|
||||
return
|
||||
}
|
||||
|
||||
// FetchTxBufBySha return the bytestream data and associated protocol version.
|
||||
// for the given Tx Sha
|
||||
func (db *LevelDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) {
|
||||
_, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha)
|
||||
return
|
||||
}
|
||||
|
||||
// InvalidateTxCache clear/release all cached transactions.
|
||||
func (db *LevelDb) InvalidateTxCache() {
|
||||
}
|
||||
|
||||
// InvalidateTxCache clear/release all cached blocks.
|
||||
func (db *LevelDb) InvalidateBlockCache() {
|
||||
}
|
||||
|
||||
// InvalidateCache clear/release all cached blocks and transactions.
|
||||
func (db *LevelDb) InvalidateCache() {
|
||||
}
|
||||
56
ldb/dbtest/dbtst.go
Normal file
56
ldb/dbtest/dbtst.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
//
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
type tst struct {
|
||||
key int
|
||||
value string
|
||||
}
|
||||
|
||||
var dataset = []tst{
|
||||
//var dataset = []struct { key int, value string } {
|
||||
{1, "one"},
|
||||
{2, "two"},
|
||||
{3, "three"},
|
||||
{4, "four"},
|
||||
{5, "five"},
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
ro := &opt.ReadOptions{}
|
||||
wo := &opt.WriteOptions{}
|
||||
|
||||
ldb, err := leveldb.OpenFile("dbfile", &opt.Options{Flag: opt.OFCreateIfMissing})
|
||||
if err != nil {
|
||||
fmt.Printf("db open failed %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
batch := new(leveldb.Batch)
|
||||
for _, datum := range dataset {
|
||||
key := fmt.Sprintf("%v", datum.key)
|
||||
batch.Put([]byte(key), []byte(datum.value))
|
||||
}
|
||||
err = ldb.Write(batch, wo)
|
||||
|
||||
for _, datum := range dataset {
|
||||
key := fmt.Sprintf("%v", datum.key)
|
||||
data, err := ldb.Get([]byte(key), ro)
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("db read failed %v\n", err)
|
||||
}
|
||||
|
||||
if string(data) != datum.value {
|
||||
fmt.Printf("mismatched data from db key %v val %v db %v", key, datum.value, data)
|
||||
}
|
||||
}
|
||||
fmt.Printf("completed\n")
|
||||
ldb.Close()
|
||||
}
|
||||
15
ldb/doc.go
Normal file
15
ldb/doc.go
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
/*
|
||||
Package sqlite3 implements a sqlite3 instance of btcdb.
|
||||
|
||||
sqlite provides a zero setup, single file database. It requires cgo
|
||||
and the presence of the sqlite library and headers, but nothing else.
|
||||
The performance is generally high although it goes down with database
|
||||
size.
|
||||
|
||||
Many of the block or tx specific functions for btcdb are in this subpackage.
|
||||
*/
|
||||
package ldb
|
||||
202
ldb/insertremove_test.go
Normal file
202
ldb/insertremove_test.go
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ldb_test
|
||||
|
||||
import (
|
||||
"github.com/conformal/btcdb"
|
||||
_ "github.com/conformal/btcdb/ldb"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var tstBlocks []*btcutil.Block
|
||||
|
||||
func loadblocks(t *testing.T) []*btcutil.Block {
|
||||
if len(tstBlocks) != 0 {
|
||||
return tstBlocks
|
||||
}
|
||||
|
||||
testdatafile := filepath.Join("testdata", "blocks1-256.bz2")
|
||||
blocks, err := loadBlocks(t, testdatafile)
|
||||
if err != nil {
|
||||
t.Errorf("Unable to load blocks from test data: %v", err)
|
||||
return nil
|
||||
}
|
||||
tstBlocks = blocks
|
||||
return blocks
|
||||
}
|
||||
|
||||
func TestUnspentInsert(t *testing.T) {
|
||||
testUnspentInsert(t, dbTmDefault)
|
||||
testUnspentInsert(t, dbTmNormal)
|
||||
testUnspentInsert(t, dbTmFast)
|
||||
}
|
||||
|
||||
// insert every block in the test chain
|
||||
// after each insert, fetch all the tx affected by the latest
|
||||
// block and verify that the the tx is spent/unspent
|
||||
// new tx should be fully unspent, referenced tx should have
|
||||
// the associated txout set to spent.
|
||||
func testUnspentInsert(t *testing.T, mode int) {
|
||||
// Ignore db remove errors since it means we didn't have an old one.
|
||||
dbname := "tstdbuspnt1"
|
||||
_ = os.RemoveAll(dbname)
|
||||
db, err := btcdb.CreateDB("leveldb", dbname)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to open test database %v", err)
|
||||
return
|
||||
}
|
||||
defer os.RemoveAll(dbname)
|
||||
defer db.Close()
|
||||
|
||||
switch mode {
|
||||
case dbTmDefault: // default
|
||||
// no setup
|
||||
case dbTmNormal: // explicit normal
|
||||
db.SetDBInsertMode(btcdb.InsertNormal)
|
||||
case dbTmFast: // fast mode
|
||||
|
||||
case dbTmNoVerify: // validated block
|
||||
t.Errorf("UnspentInsert test is not valid in NoVerify mode")
|
||||
}
|
||||
|
||||
// Since we are dealing with small dataset, reduce cache size
|
||||
|
||||
blocks := loadblocks(t)
|
||||
endtest:
|
||||
for height := int64(0); height < int64(len(blocks)); height++ {
|
||||
|
||||
block := blocks[height]
|
||||
// look up inputs to this x
|
||||
mblock := block.MsgBlock()
|
||||
var txneededList []*btcwire.ShaHash
|
||||
var txlookupList []*btcwire.ShaHash
|
||||
var txOutList []*btcwire.ShaHash
|
||||
var txInList []*btcwire.OutPoint
|
||||
for _, tx := range mblock.Transactions {
|
||||
for _, txin := range tx.TxIn {
|
||||
if txin.PreviousOutpoint.Index == uint32(4294967295) {
|
||||
continue
|
||||
}
|
||||
origintxsha := &txin.PreviousOutpoint.Hash
|
||||
|
||||
txInList = append(txInList, &txin.PreviousOutpoint)
|
||||
txneededList = append(txneededList, origintxsha)
|
||||
txlookupList = append(txlookupList, origintxsha)
|
||||
|
||||
if !db.ExistsTxSha(origintxsha) {
|
||||
t.Errorf("referenced tx not found %v ", origintxsha)
|
||||
}
|
||||
|
||||
}
|
||||
txshaname, _ := tx.TxSha()
|
||||
txlookupList = append(txlookupList, &txshaname)
|
||||
txOutList = append(txOutList, &txshaname)
|
||||
}
|
||||
|
||||
txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{}
|
||||
txlist := db.FetchTxByShaList(txneededList)
|
||||
for _, txe := range txlist {
|
||||
if txe.Err != nil {
|
||||
t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err)
|
||||
break endtest
|
||||
}
|
||||
txneededmap[*txe.Sha] = txe
|
||||
}
|
||||
for _, spend := range txInList {
|
||||
itxe := txneededmap[spend.Hash]
|
||||
if itxe.TxSpent[spend.Index] == true {
|
||||
t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index)
|
||||
}
|
||||
}
|
||||
|
||||
newheight, err := db.InsertBlock(block)
|
||||
if err != nil {
|
||||
t.Errorf("failed to insert block %v err %v", height, err)
|
||||
break endtest
|
||||
}
|
||||
if newheight != height {
|
||||
t.Errorf("height mismatch expect %v returned %v", height, newheight)
|
||||
break endtest
|
||||
}
|
||||
|
||||
txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{}
|
||||
txlist = db.FetchTxByShaList(txlookupList)
|
||||
for _, txe := range txlist {
|
||||
if txe.Err != nil {
|
||||
t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err)
|
||||
break endtest
|
||||
}
|
||||
txlookupmap[*txe.Sha] = txe
|
||||
}
|
||||
for _, spend := range txInList {
|
||||
itxe := txlookupmap[spend.Hash]
|
||||
if itxe.TxSpent[spend.Index] == false {
|
||||
t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent)
|
||||
}
|
||||
}
|
||||
for _, txo := range txOutList {
|
||||
itxe := txlookupmap[*txo]
|
||||
for i, spent := range itxe.TxSpent {
|
||||
if spent == true {
|
||||
t.Errorf("freshly inserted tx %v already spent %v", txo, i)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if len(txInList) == 0 {
|
||||
continue
|
||||
}
|
||||
dropblock := blocks[height-1]
|
||||
dropsha, _ := dropblock.Sha()
|
||||
|
||||
err = db.DropAfterBlockBySha(dropsha)
|
||||
if err != nil {
|
||||
t.Errorf("failed to drop block %v err %v", height, err)
|
||||
break endtest
|
||||
}
|
||||
|
||||
txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{}
|
||||
txlist = db.FetchTxByShaList(txlookupList)
|
||||
for _, txe := range txlist {
|
||||
if txe.Err != nil {
|
||||
if _, ok := txneededmap[*txe.Sha]; ok {
|
||||
t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err)
|
||||
break endtest
|
||||
}
|
||||
}
|
||||
txlookupmap[*txe.Sha] = txe
|
||||
}
|
||||
for _, spend := range txInList {
|
||||
itxe := txlookupmap[spend.Hash]
|
||||
if itxe.TxSpent[spend.Index] == true {
|
||||
t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent)
|
||||
}
|
||||
}
|
||||
newheight, err = db.InsertBlock(block)
|
||||
if err != nil {
|
||||
t.Errorf("failed to insert block %v err %v", height, err)
|
||||
break endtest
|
||||
}
|
||||
txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{}
|
||||
txlist = db.FetchTxByShaList(txlookupList)
|
||||
for _, txe := range txlist {
|
||||
if txe.Err != nil {
|
||||
t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err)
|
||||
break endtest
|
||||
}
|
||||
txlookupmap[*txe.Sha] = txe
|
||||
}
|
||||
for _, spend := range txInList {
|
||||
itxe := txlookupmap[spend.Hash]
|
||||
if itxe.TxSpent[spend.Index] == false {
|
||||
t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
24
ldb/internal_test.go
Normal file
24
ldb/internal_test.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/conformal/btcdb"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
// FetchSha returns the datablock and pver for the given ShaHash.
|
||||
// This is a testing only interface.
|
||||
func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32,
|
||||
blkid int64, err error) {
|
||||
sqldb, ok := db.(*LevelDb)
|
||||
if !ok {
|
||||
err = fmt.Errorf("Invalid data type")
|
||||
return
|
||||
}
|
||||
buf, blkid, err = sqldb.fetchSha(sha)
|
||||
return
|
||||
}
|
||||
528
ldb/leveldb.go
Normal file
528
ldb/leveldb.go
Normal file
|
|
@ -0,0 +1,528 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/conformal/btcdb"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
"github.com/conformal/seelog"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
dbVersion int = 2
|
||||
dbMaxTransCnt = 20000
|
||||
dbMaxTransMem = 64 * 1024 * 1024 // 64 MB
|
||||
)
|
||||
|
||||
var log seelog.LoggerInterface = seelog.Disabled
|
||||
|
||||
type tBlockInsertData struct {
|
||||
sha btcwire.ShaHash
|
||||
pver uint32
|
||||
buf []byte
|
||||
}
|
||||
type tTxInsertData struct {
|
||||
txsha *btcwire.ShaHash
|
||||
blockid int64
|
||||
txoff int
|
||||
txlen int
|
||||
usedbuf []byte
|
||||
}
|
||||
|
||||
type LevelDb struct {
|
||||
// lock preventing multiple entry
|
||||
dbLock sync.Mutex
|
||||
|
||||
// leveldb pieces
|
||||
bShaDb *leveldb.DB
|
||||
bBlkDb *leveldb.DB
|
||||
tShaDb *leveldb.DB
|
||||
ro *opt.ReadOptions
|
||||
wo *opt.WriteOptions
|
||||
|
||||
bShabatch *leveldb.Batch
|
||||
bBlkbatch *leveldb.Batch
|
||||
|
||||
nextBlock int64
|
||||
|
||||
lastBlkShaCached bool
|
||||
lastBlkSha btcwire.ShaHash
|
||||
lastBlkIdx int64
|
||||
|
||||
txUpdateMap map[btcwire.ShaHash]*txUpdateObj
|
||||
}
|
||||
|
||||
var self = btcdb.DriverDB{DbType: "leveldb", Create: CreateDB, Open: OpenDB}
|
||||
|
||||
func init() {
|
||||
btcdb.AddDBDriver(self)
|
||||
}
|
||||
|
||||
// OpenDB opens an existing database for use.
|
||||
func OpenDB(dbpath string) (btcdb.Db, error) {
|
||||
log = btcdb.GetLog()
|
||||
|
||||
db, err := openDB(dbpath, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Info("Opening DB\n")
|
||||
|
||||
// Need to find last block and tx
|
||||
|
||||
var lastknownblock, nextunknownblock, testblock int64
|
||||
|
||||
increment := int64(100000)
|
||||
ldb := db.(*LevelDb)
|
||||
|
||||
var lastSha *btcwire.ShaHash
|
||||
// forward scan
|
||||
blockforward:
|
||||
for {
|
||||
|
||||
sha, _, err := ldb.getBlkByHeight(testblock)
|
||||
if err == nil {
|
||||
// block is found
|
||||
lastSha = sha
|
||||
lastknownblock = testblock
|
||||
testblock += increment
|
||||
} else {
|
||||
if testblock == 0 {
|
||||
//no blocks in db, odd but ok.
|
||||
return db, nil
|
||||
}
|
||||
nextunknownblock = testblock
|
||||
break blockforward
|
||||
}
|
||||
}
|
||||
|
||||
// narrow search
|
||||
blocknarrow:
|
||||
for {
|
||||
testblock = (lastknownblock + nextunknownblock) / 2
|
||||
sha, _, err := ldb.getBlkByHeight(testblock)
|
||||
if err == nil {
|
||||
lastknownblock = testblock
|
||||
lastSha = sha
|
||||
} else {
|
||||
nextunknownblock = testblock
|
||||
}
|
||||
if lastknownblock+1 == nextunknownblock {
|
||||
break blocknarrow
|
||||
}
|
||||
}
|
||||
|
||||
ldb.lastBlkSha = *lastSha
|
||||
ldb.lastBlkIdx = lastknownblock
|
||||
ldb.nextBlock = lastknownblock + 1
|
||||
|
||||
log.Info("Opening DB: completed\n")
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) {
|
||||
var db LevelDb
|
||||
var tbShaDb, ttShaDb, tbBlkDb *leveldb.DB
|
||||
defer func() {
|
||||
if err == nil {
|
||||
db.bShaDb = tbShaDb
|
||||
db.bBlkDb = tbBlkDb
|
||||
db.tShaDb = ttShaDb
|
||||
|
||||
db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{}
|
||||
|
||||
pbdb = &db
|
||||
}
|
||||
}()
|
||||
|
||||
if flag&opt.OFCreateIfMissing == opt.OFCreateIfMissing {
|
||||
err = os.Mkdir(dbpath, 0750)
|
||||
if err != nil {
|
||||
log.Errorf("mkdir failed %v %v", dbpath, err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
_, err = os.Stat(dbpath)
|
||||
if err != nil {
|
||||
err = btcdb.DbDoesNotExist
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
bShaName := filepath.Join(dbpath, "bSha.ldb")
|
||||
tbShaDb, err = leveldb.OpenFile(bShaName, &opt.Options{Flag: flag})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
bBlkName := filepath.Join(dbpath, "bBlk.ldb")
|
||||
tbBlkDb, err = leveldb.OpenFile(bBlkName, &opt.Options{Flag: flag})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tShaName := filepath.Join(dbpath, "tSha.ldb")
|
||||
ttShaDb, err = leveldb.OpenFile(tShaName, &opt.Options{Flag: flag})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// CreateDB creates, initializes and opens a database for use.
|
||||
func CreateDB(dbpath string) (btcdb.Db, error) {
|
||||
log = btcdb.GetLog()
|
||||
|
||||
// No special setup needed, just OpenBB
|
||||
return openDB(dbpath, opt.OFCreateIfMissing)
|
||||
}
|
||||
|
||||
func (db *LevelDb) close() {
|
||||
db.bShaDb.Close()
|
||||
db.bBlkDb.Close()
|
||||
db.tShaDb.Close()
|
||||
}
|
||||
|
||||
// Sync verifies that the database is coherent on disk,
|
||||
// and no outstanding transactions are in flight.
|
||||
func (db *LevelDb) Sync() {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
// while specified by the API, does nothing
|
||||
// however does grab lock to verify it does not return until other operations are complete.
|
||||
}
|
||||
|
||||
// Close cleanly shuts down database, syncing all data.
|
||||
func (db *LevelDb) Close() {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
db.close()
|
||||
}
|
||||
|
||||
// DropAfterBlockBySha will remove any blocks from the database after
|
||||
// the given block.
|
||||
func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
defer db.processBatches()
|
||||
|
||||
startheight := db.nextBlock - 1
|
||||
|
||||
keepidx, err := db.getBlkLoc(sha)
|
||||
if err != nil {
|
||||
// should the error here be normalized ?
|
||||
log.Infof("block loc failed %v ", sha)
|
||||
return err
|
||||
}
|
||||
|
||||
for height := startheight; height > keepidx; height = height - 1 {
|
||||
var blk *btcutil.Block
|
||||
blksha, buf, err := db.getBlkByHeight(height)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blk, err = btcutil.NewBlockFromBytes(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, tx := range blk.MsgBlock().Transactions {
|
||||
err = db.unSpend(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// rather than iterate the list of tx backward, do it twice.
|
||||
for _, tx := range blk.MsgBlock().Transactions {
|
||||
txSha, _ := tx.TxSha()
|
||||
var txUo txUpdateObj
|
||||
txUo.delete = true
|
||||
db.txUpdateMap[txSha] = &txUo
|
||||
}
|
||||
db.bShaBatch().Delete(shaToKey(blksha))
|
||||
db.bBlkBatch().Delete(int64ToKey(height))
|
||||
}
|
||||
|
||||
db.nextBlock = keepidx + 1
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertBlock inserts raw block and transaction data from a block into the
|
||||
// database. The first block inserted into the database will be treated as the
|
||||
// genesis block. Every subsequent block insert requires the referenced parent
|
||||
// block to already exist.
|
||||
func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
defer db.processBatches()
|
||||
|
||||
blocksha, err := block.Sha()
|
||||
if err != nil {
|
||||
log.Warnf("Failed to compute block sha %v", blocksha)
|
||||
return
|
||||
}
|
||||
mblock := block.MsgBlock()
|
||||
rawMsg, err := block.Bytes()
|
||||
if err != nil {
|
||||
log.Warnf("Failed to obtain raw block sha %v", blocksha)
|
||||
return
|
||||
}
|
||||
txloc, err := block.TxLoc()
|
||||
if err != nil {
|
||||
log.Warnf("Failed to obtain raw block sha %v", blocksha)
|
||||
return
|
||||
}
|
||||
|
||||
// Insert block into database
|
||||
newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock,
|
||||
rawMsg)
|
||||
if err != nil {
|
||||
log.Warnf("Failed to insert block %v %v %v", blocksha,
|
||||
&mblock.Header.PrevBlock, err)
|
||||
return
|
||||
}
|
||||
|
||||
// At least two blocks in the long past were generated by faulty
|
||||
// miners, the sha of the transaction exists in a previous block,
|
||||
// detect this condition and 'accept' the block.
|
||||
for txidx, tx := range mblock.Transactions {
|
||||
var txsha btcwire.ShaHash
|
||||
txsha, err = tx.TxSha()
|
||||
if err != nil {
|
||||
log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err)
|
||||
return
|
||||
}
|
||||
// Some old blocks contain duplicate transactions
|
||||
// Attempt to cleanly bypass this problem
|
||||
// http://blockexplorer.com/b/91842
|
||||
// http://blockexplorer.com/b/91880
|
||||
if newheight == 91842 {
|
||||
dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599")
|
||||
if err != nil {
|
||||
panic("invalid sha string in source")
|
||||
}
|
||||
if txsha == *dupsha {
|
||||
//log.Tracef("skipping sha %v %v", dupsha, newheight)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if newheight == 91880 {
|
||||
dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468")
|
||||
if err != nil {
|
||||
panic("invalid sha string in source")
|
||||
}
|
||||
if txsha == *dupsha {
|
||||
//log.Tracef("skipping sha %v %v", dupsha, newheight)
|
||||
continue
|
||||
}
|
||||
}
|
||||
spentbuflen := (len(tx.TxOut) + 7) / 8
|
||||
spentbuf := make([]byte, spentbuflen, spentbuflen)
|
||||
if len(tx.TxOut)%8 != 0 {
|
||||
for i := uint(len(tx.TxOut) % 8); i < 8; i++ {
|
||||
spentbuf[spentbuflen-1] |= (byte(1) << i)
|
||||
}
|
||||
}
|
||||
|
||||
err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf)
|
||||
if err != nil {
|
||||
log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err)
|
||||
|
||||
return
|
||||
}
|
||||
err = db.doSpend(tx)
|
||||
if err != nil {
|
||||
log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
return newheight, nil
|
||||
}
|
||||
|
||||
// SetDBInsertMode provides hints to the database to how the application
|
||||
// is running this allows the database to work in optimized modes when the
|
||||
// database may be very busy.
|
||||
func (db *LevelDb) SetDBInsertMode(newmode btcdb.InsertMode) {
|
||||
|
||||
// special modes are not supported
|
||||
}
|
||||
|
||||
// doSpend iterates all TxIn in a bitcoin transaction marking each associated
|
||||
// TxOut as spent.
|
||||
func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error {
|
||||
for txinidx := range tx.TxIn {
|
||||
txin := tx.TxIn[txinidx]
|
||||
|
||||
inTxSha := txin.PreviousOutpoint.Hash
|
||||
inTxidx := txin.PreviousOutpoint.Index
|
||||
|
||||
if inTxidx == ^uint32(0) {
|
||||
continue
|
||||
}
|
||||
|
||||
//log.Infof("spending %v %v", &inTxSha, inTxidx)
|
||||
|
||||
err := db.setSpentData(&inTxSha, inTxidx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unSpend iterates all TxIn in a bitcoin transaction marking each associated
|
||||
// TxOut as unspent.
|
||||
func (db *LevelDb) unSpend(tx *btcwire.MsgTx) error {
|
||||
for txinidx := range tx.TxIn {
|
||||
txin := tx.TxIn[txinidx]
|
||||
|
||||
inTxSha := txin.PreviousOutpoint.Hash
|
||||
inTxidx := txin.PreviousOutpoint.Index
|
||||
|
||||
if inTxidx == ^uint32(0) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := db.clearSpentData(&inTxSha, inTxidx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *LevelDb) setSpentData(sha *btcwire.ShaHash, idx uint32) error {
|
||||
return db.setclearSpentData(sha, idx, true)
|
||||
}
|
||||
|
||||
func (db *LevelDb) clearSpentData(sha *btcwire.ShaHash, idx uint32) error {
|
||||
return db.setclearSpentData(sha, idx, false)
|
||||
}
|
||||
|
||||
func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bool) error {
|
||||
var txUo *txUpdateObj
|
||||
var ok bool
|
||||
|
||||
if txUo, ok = db.txUpdateMap[*txsha]; !ok {
|
||||
// not cached, load from db
|
||||
var txU txUpdateObj
|
||||
blkHeight, txOff, txLen, spentData, err := db.getTxData(txsha)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
txU.txSha = txsha
|
||||
txU.blkHeight = blkHeight
|
||||
txU.txoff = txOff
|
||||
txU.txlen = txLen
|
||||
txU.spentData = spentData
|
||||
|
||||
txUo = &txU
|
||||
}
|
||||
|
||||
byteidx := idx / 8
|
||||
byteoff := idx % 8
|
||||
|
||||
if set {
|
||||
txUo.spentData[byteidx] |= (byte(1) << byteoff)
|
||||
} else {
|
||||
txUo.spentData[byteidx] &= ^(byte(1) << byteoff)
|
||||
}
|
||||
|
||||
db.txUpdateMap[*txsha] = txUo
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func intToKey(keyint int) []byte {
|
||||
key := fmt.Sprintf("%d", keyint)
|
||||
return []byte(key)
|
||||
}
|
||||
func int64ToKey(keyint int64) []byte {
|
||||
key := fmt.Sprintf("%d", keyint)
|
||||
return []byte(key)
|
||||
}
|
||||
|
||||
func shaToKey(sha *btcwire.ShaHash) []byte {
|
||||
return sha.Bytes()
|
||||
}
|
||||
|
||||
func (db *LevelDb) bShaBatch() *leveldb.Batch {
|
||||
if db.bShabatch == nil {
|
||||
db.bShabatch = new(leveldb.Batch)
|
||||
}
|
||||
return db.bShabatch
|
||||
}
|
||||
|
||||
func (db *LevelDb) bBlkBatch() *leveldb.Batch {
|
||||
if db.bBlkbatch == nil {
|
||||
db.bBlkbatch = new(leveldb.Batch)
|
||||
}
|
||||
return db.bBlkbatch
|
||||
}
|
||||
|
||||
func (db *LevelDb) processBatches() {
|
||||
var err error
|
||||
if db.bShabatch != nil {
|
||||
err = db.bShaDb.Write(db.bShabatch, db.wo)
|
||||
db.bShabatch.Reset()
|
||||
db.bShabatch = nil
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if db.bBlkbatch != nil {
|
||||
err = db.bBlkDb.Write(db.bBlkbatch, db.wo)
|
||||
db.bBlkbatch.Reset()
|
||||
db.bBlkbatch = nil
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if len(db.txUpdateMap) != 0 {
|
||||
tShabatch := new(leveldb.Batch)
|
||||
|
||||
for txSha, txU := range db.txUpdateMap {
|
||||
key := shaToKey(&txSha)
|
||||
if txU.delete {
|
||||
//log.Infof("deleting tx %v", txSha)
|
||||
tShabatch.Delete(key)
|
||||
} else {
|
||||
//log.Infof("inserting tx %v", txSha)
|
||||
txdat, err := db.formatTx(txU)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tShabatch.Put(key, txdat)
|
||||
}
|
||||
}
|
||||
|
||||
err = db.tShaDb.Write(tShabatch, db.wo)
|
||||
tShabatch.Reset()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{}
|
||||
runtime.GC()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (db *LevelDb) RollbackClose() {
|
||||
db.close()
|
||||
}
|
||||
394
ldb/operational_test.go
Normal file
394
ldb/operational_test.go
Normal file
|
|
@ -0,0 +1,394 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ldb_test
|
||||
|
||||
import (
|
||||
"compress/bzip2"
|
||||
"encoding/binary"
|
||||
"github.com/conformal/btcdb"
|
||||
"github.com/conformal/btcutil"
|
||||
"github.com/conformal/btcwire"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var network = btcwire.MainNet
|
||||
|
||||
const (
|
||||
dbTmDefault = iota
|
||||
dbTmNormal
|
||||
dbTmFast
|
||||
dbTmNoVerify
|
||||
)
|
||||
|
||||
func TestOperational(t *testing.T) {
|
||||
testOperationalMode(t, dbTmDefault)
|
||||
testOperationalMode(t, dbTmNormal)
|
||||
testOperationalMode(t, dbTmFast)
|
||||
testOperationalMode(t, dbTmNoVerify)
|
||||
}
|
||||
|
||||
func testOperationalMode(t *testing.T, mode int) {
|
||||
// simplified basic operation is:
|
||||
// 1) fetch block from remote server
|
||||
// 2) look up all txin (except coinbase in db)
|
||||
// 3) insert block
|
||||
|
||||
// Ignore db remove errors since it means we didn't have an old one.
|
||||
dbname := "tstdbop1"
|
||||
_ = os.RemoveAll(dbname)
|
||||
db, err := btcdb.CreateDB("leveldb", dbname)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to open test database %v", err)
|
||||
return
|
||||
}
|
||||
defer os.RemoveAll(dbname)
|
||||
defer db.Close()
|
||||
|
||||
switch mode {
|
||||
case dbTmDefault: // default
|
||||
// no setup
|
||||
case dbTmNormal: // explicit normal
|
||||
db.SetDBInsertMode(btcdb.InsertNormal)
|
||||
case dbTmFast: // fast mode
|
||||
|
||||
case dbTmNoVerify: // validated block
|
||||
db.SetDBInsertMode(btcdb.InsertValidatedInput)
|
||||
}
|
||||
|
||||
testdatafile := filepath.Join("testdata", "blocks1-256.bz2")
|
||||
blocks, err := loadBlocks(t, testdatafile)
|
||||
if err != nil {
|
||||
t.Errorf("Unable to load blocks from test data for mode %v: %v",
|
||||
mode, err)
|
||||
return
|
||||
}
|
||||
|
||||
err = nil
|
||||
out:
|
||||
for height := int64(0); height < int64(len(blocks)); height++ {
|
||||
block := blocks[height]
|
||||
if mode != dbTmNoVerify {
|
||||
// except for NoVerify which does not allow lookups check inputs
|
||||
mblock := block.MsgBlock()
|
||||
var txneededList []*btcwire.ShaHash
|
||||
for _, tx := range mblock.Transactions {
|
||||
for _, txin := range tx.TxIn {
|
||||
if txin.PreviousOutpoint.Index == uint32(4294967295) {
|
||||
continue
|
||||
}
|
||||
origintxsha := &txin.PreviousOutpoint.Hash
|
||||
txneededList = append(txneededList, origintxsha)
|
||||
|
||||
if !db.ExistsTxSha(origintxsha) {
|
||||
t.Errorf("referenced tx not found %v ", origintxsha)
|
||||
}
|
||||
|
||||
_, _, _, _, err := db.FetchTxAllBySha(origintxsha)
|
||||
if err != nil {
|
||||
t.Errorf("referenced tx not found %v err %v ", origintxsha, err)
|
||||
}
|
||||
_, _, _, _, err = db.FetchTxAllBySha(origintxsha)
|
||||
if err != nil {
|
||||
t.Errorf("referenced tx not found %v err %v ", origintxsha, err)
|
||||
}
|
||||
_, _, _, err = db.FetchTxBySha(origintxsha)
|
||||
if err != nil {
|
||||
t.Errorf("referenced tx not found %v err %v ", origintxsha, err)
|
||||
}
|
||||
_, _, err = db.FetchTxBufBySha(origintxsha)
|
||||
if err != nil {
|
||||
t.Errorf("referenced tx not found %v err %v ", origintxsha, err)
|
||||
}
|
||||
_, err = db.FetchTxUsedBySha(origintxsha)
|
||||
if err != nil {
|
||||
t.Errorf("tx used fetch fail %v err %v ", origintxsha, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
txlist := db.FetchTxByShaList(txneededList)
|
||||
for _, txe := range txlist {
|
||||
if txe.Err != nil {
|
||||
t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err)
|
||||
break out
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
newheight, err := db.InsertBlock(block)
|
||||
if err != nil {
|
||||
t.Errorf("failed to insert block %v err %v", height, err)
|
||||
break out
|
||||
}
|
||||
if newheight != height {
|
||||
t.Errorf("height mismatch expect %v returned %v", height, newheight)
|
||||
break out
|
||||
}
|
||||
|
||||
newSha, blkid, err := db.NewestSha()
|
||||
if err != nil {
|
||||
t.Errorf("failed to obtain latest sha %v %v", height, err)
|
||||
}
|
||||
|
||||
if blkid != height {
|
||||
t.Errorf("height doe not match latest block height %v %v", blkid, height, err)
|
||||
}
|
||||
|
||||
blkSha, _ := block.Sha()
|
||||
if *newSha != *blkSha {
|
||||
t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err)
|
||||
}
|
||||
}
|
||||
|
||||
// now that db is populated, do some additional test
|
||||
testFetchRangeHeight(t, db, blocks)
|
||||
|
||||
switch mode {
|
||||
case dbTmDefault: // default
|
||||
// no cleanup
|
||||
case dbTmNormal: // explicit normal
|
||||
// no cleanup
|
||||
case dbTmFast: // fast mode
|
||||
db.SetDBInsertMode(btcdb.InsertNormal)
|
||||
case dbTmNoVerify: // validated block
|
||||
db.SetDBInsertMode(btcdb.InsertNormal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackout(t *testing.T) {
|
||||
testBackout(t, dbTmDefault)
|
||||
testBackout(t, dbTmNormal)
|
||||
testBackout(t, dbTmFast)
|
||||
}
|
||||
|
||||
func testBackout(t *testing.T, mode int) {
|
||||
// simplified basic operation is:
|
||||
// 1) fetch block from remote server
|
||||
// 2) look up all txin (except coinbase in db)
|
||||
// 3) insert block
|
||||
|
||||
// Ignore db remove errors since it means we didn't have an old one.
|
||||
dbname := "tstdbop2"
|
||||
_ = os.RemoveAll(dbname)
|
||||
db, err := btcdb.CreateDB("leveldb", dbname)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to open test database %v", err)
|
||||
return
|
||||
}
|
||||
defer os.RemoveAll(dbname)
|
||||
defer db.Close()
|
||||
|
||||
switch mode {
|
||||
case dbTmDefault: // default
|
||||
// no setup
|
||||
case dbTmNormal: // explicit normal
|
||||
db.SetDBInsertMode(btcdb.InsertNormal)
|
||||
case dbTmFast: // fast mode
|
||||
|
||||
}
|
||||
|
||||
testdatafile := filepath.Join("testdata", "blocks1-256.bz2")
|
||||
blocks, err := loadBlocks(t, testdatafile)
|
||||
if len(blocks) < 120 {
|
||||
t.Errorf("test data too small")
|
||||
return
|
||||
}
|
||||
|
||||
err = nil
|
||||
for height := int64(0); height < int64(len(blocks)); height++ {
|
||||
if height == 100 {
|
||||
t.Logf("Syncing at block height 100")
|
||||
db.Sync()
|
||||
}
|
||||
if height == 120 {
|
||||
t.Logf("Simulating unexpected application quit")
|
||||
// Simulate unexpected application quit
|
||||
db.RollbackClose()
|
||||
break
|
||||
}
|
||||
|
||||
block := blocks[height]
|
||||
|
||||
newheight, err := db.InsertBlock(block)
|
||||
if err != nil {
|
||||
t.Errorf("failed to insert block %v err %v", height, err)
|
||||
break
|
||||
}
|
||||
if newheight != height {
|
||||
t.Errorf("height mismatch expect %v returned %v", height, newheight)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// db was closed at height 120, so no cleanup is possible.
|
||||
|
||||
// reopen db
|
||||
db, err = btcdb.OpenDB("leveldb", dbname)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to open test database %v", err)
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
sha, err := blocks[99].Sha()
|
||||
if err != nil {
|
||||
t.Errorf("failed to get block 99 sha err %v", err)
|
||||
return
|
||||
}
|
||||
_ = db.ExistsSha(sha)
|
||||
_, err = db.FetchBlockBySha(sha)
|
||||
if err != nil {
|
||||
t.Errorf("failed to load block 99 from db %v", err)
|
||||
}
|
||||
|
||||
sha, err = blocks[119].Sha()
|
||||
if err != nil {
|
||||
t.Errorf("failed to get block 110 sha err %v", err)
|
||||
return
|
||||
}
|
||||
_ = db.ExistsSha(sha)
|
||||
_, err = db.FetchBlockBySha(sha)
|
||||
if err != nil {
|
||||
t.Errorf("loaded block 119 from db")
|
||||
return
|
||||
}
|
||||
|
||||
block := blocks[119]
|
||||
mblock := block.MsgBlock()
|
||||
txsha, err := mblock.Transactions[0].TxSha()
|
||||
exists := db.ExistsTxSha(&txsha)
|
||||
if !exists {
|
||||
t.Errorf("tx %v not located db\n", txsha)
|
||||
}
|
||||
|
||||
_, _, _, err = db.FetchTxBySha(&txsha)
|
||||
if err != nil {
|
||||
t.Errorf("tx %v not located db\n", txsha)
|
||||
return
|
||||
}
|
||||
_, err = db.FetchTxUsedBySha(&txsha)
|
||||
if err != nil {
|
||||
t.Errorf("tx %v not located db\n", txsha)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) {
|
||||
testdatafile := filepath.Join("testdata", "blocks1-256.bz2")
|
||||
var dr io.Reader
|
||||
var fi io.ReadCloser
|
||||
fi, err = os.Open(testdatafile)
|
||||
if err != nil {
|
||||
t.Errorf("failed to open file %v, err %v", testdatafile, err)
|
||||
return
|
||||
}
|
||||
if strings.HasSuffix(testdatafile, ".bz2") {
|
||||
z := bzip2.NewReader(fi)
|
||||
dr = z
|
||||
} else {
|
||||
dr = fi
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := fi.Close(); err != nil {
|
||||
t.Errorf("failed to close file %v %v", testdatafile, err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Set the first block as the genesis block.
|
||||
genesis := btcutil.NewBlock(&btcwire.GenesisBlock)
|
||||
blocks = append(blocks, genesis)
|
||||
|
||||
var block *btcutil.Block
|
||||
err = nil
|
||||
for height := int64(1); err == nil; height++ {
|
||||
var rintbuf uint32
|
||||
err = binary.Read(dr, binary.LittleEndian, &rintbuf)
|
||||
if err == io.EOF {
|
||||
// hit end of file at expected offset: no warning
|
||||
height--
|
||||
err = nil
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("failed to load network type, err %v", err)
|
||||
break
|
||||
}
|
||||
if rintbuf != uint32(network) {
|
||||
t.Errorf("Block doesn't match network: %v expects %v",
|
||||
rintbuf, network)
|
||||
break
|
||||
}
|
||||
err = binary.Read(dr, binary.LittleEndian, &rintbuf)
|
||||
blocklen := rintbuf
|
||||
|
||||
rbytes := make([]byte, blocklen)
|
||||
|
||||
// read block
|
||||
dr.Read(rbytes)
|
||||
|
||||
block, err = btcutil.NewBlockFromBytes(rbytes)
|
||||
if err != nil {
|
||||
t.Errorf("failed to parse block %v", height)
|
||||
return
|
||||
}
|
||||
blocks = append(blocks, block)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) {
|
||||
|
||||
var testincrement int64 = 50
|
||||
var testcnt int64 = 100
|
||||
|
||||
shanames := make([]*btcwire.ShaHash, len(blocks))
|
||||
|
||||
nBlocks := int64(len(blocks))
|
||||
|
||||
for i := range blocks {
|
||||
blockSha, err := blocks[i].Sha()
|
||||
if err != nil {
|
||||
t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err)
|
||||
}
|
||||
shanames[i] = blockSha
|
||||
}
|
||||
|
||||
for startheight := int64(0); startheight < nBlocks; startheight += testincrement {
|
||||
endheight := startheight + testcnt
|
||||
|
||||
if endheight > nBlocks {
|
||||
endheight = btcdb.AllShas
|
||||
}
|
||||
|
||||
shalist, err := db.FetchHeightRange(startheight, endheight)
|
||||
if err != nil {
|
||||
t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err)
|
||||
}
|
||||
|
||||
if endheight == btcdb.AllShas {
|
||||
if int64(len(shalist)) != nBlocks-startheight {
|
||||
t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist))
|
||||
}
|
||||
} else {
|
||||
if int64(len(shalist)) != testcnt {
|
||||
t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist))
|
||||
}
|
||||
}
|
||||
|
||||
for i := range shalist {
|
||||
sha0 := *shanames[int64(i)+startheight]
|
||||
sha1 := shalist[i]
|
||||
if sha0 != sha1 {
|
||||
t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v: %v %v ", int64(i)+startheight, startheight, endheight, sha0, sha1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
BIN
ldb/testdata/blocks1-256.bz2
vendored
Normal file
BIN
ldb/testdata/blocks1-256.bz2
vendored
Normal file
Binary file not shown.
175
ldb/tx.go
Normal file
175
ldb/tx.go
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
// Copyright (c) 2013 Conformal Systems LLC.
|
||||
// Use of this source code is governed by an ISC
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
//"github.com/conformal/btcdb"
|
||||
"github.com/conformal/btcwire"
|
||||
)
|
||||
|
||||
type txUpdateObj struct {
|
||||
txSha *btcwire.ShaHash
|
||||
blkHeight int64
|
||||
txoff int
|
||||
txlen int
|
||||
spentData []byte
|
||||
delete bool
|
||||
}
|
||||
|
||||
// InsertTx inserts a tx hash and its associated data into the database.
|
||||
func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
return db.insertTx(txsha, height, txoff, txlen, spentbuf)
|
||||
}
|
||||
|
||||
// insertTx inserts a tx hash and its associated data into the database.
|
||||
// Must be called with db lock held.
|
||||
func (db *LevelDb) insertTx(txSha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) {
|
||||
var txU txUpdateObj
|
||||
|
||||
txU.txSha = txSha
|
||||
txU.blkHeight = height
|
||||
txU.txoff = txoff
|
||||
txU.txlen = txlen
|
||||
txU.spentData = spentbuf
|
||||
|
||||
db.txUpdateMap[*txSha] = &txU
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatTx generates the value buffer for the Tx db.
|
||||
func (db *LevelDb) formatTx(txu *txUpdateObj) ([]byte, error) {
|
||||
|
||||
blkHeight := txu.blkHeight
|
||||
txoff := txu.txoff
|
||||
txlen := txu.txlen
|
||||
spentbuf := txu.spentData
|
||||
|
||||
txOff := int32(txoff)
|
||||
txLen := int32(txlen)
|
||||
|
||||
var txW bytes.Buffer
|
||||
|
||||
err := binary.Write(&txW, binary.LittleEndian, blkHeight)
|
||||
if err != nil {
|
||||
fmt.Printf("fail encoding blkHeight %v\n", err)
|
||||
err = fmt.Errorf("Write fail")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = binary.Write(&txW, binary.LittleEndian, txOff)
|
||||
if err != nil {
|
||||
fmt.Printf("fail encoding txoff %v\n", err)
|
||||
err = fmt.Errorf("Write fail")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = binary.Write(&txW, binary.LittleEndian, txLen)
|
||||
if err != nil {
|
||||
fmt.Printf("fail encoding txlen %v\n", err)
|
||||
err = fmt.Errorf("Write fail")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = binary.Write(&txW, binary.LittleEndian, spentbuf)
|
||||
if err != nil {
|
||||
fmt.Printf("fail encoding spentbuf %v\n", err)
|
||||
err = fmt.Errorf("Write fail")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return txW.Bytes(), nil
|
||||
}
|
||||
|
||||
func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (rblkHeight int64,
|
||||
rtxOff int, rtxLen int, rspentBuf []byte, err error) {
|
||||
var buf []byte
|
||||
|
||||
key := shaToKey(txsha)
|
||||
buf, err = db.tShaDb.Get(key, db.ro)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var blkHeight int64
|
||||
var txOff, txLen int32
|
||||
dr := bytes.NewBuffer(buf)
|
||||
err = binary.Read(dr, binary.LittleEndian, &blkHeight)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Db Corrupt 1")
|
||||
return
|
||||
}
|
||||
err = binary.Read(dr, binary.LittleEndian, &txOff)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Db Corrupt 2")
|
||||
return
|
||||
}
|
||||
err = binary.Read(dr, binary.LittleEndian, &txLen)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Db Corrupt 3")
|
||||
return
|
||||
}
|
||||
// remainder of buffer is spentbuf
|
||||
spentBuf := make([]byte, dr.Len())
|
||||
err = binary.Read(dr, binary.LittleEndian, spentBuf)
|
||||
if err != nil {
|
||||
fmt.Printf("fail encoding spentbuf %v\n", err)
|
||||
err = fmt.Errorf("Db Corrupt 4")
|
||||
return
|
||||
}
|
||||
return blkHeight, int(txOff), int(txLen), spentBuf, nil
|
||||
}
|
||||
|
||||
// ExistsTxSha returns if the given tx sha exists in the database
|
||||
func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
return db.existsTxSha(txsha)
|
||||
}
|
||||
|
||||
// existsTxSha returns if the given tx sha exists in the database.o
|
||||
// Must be called with the db lock held.
|
||||
func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) {
|
||||
_, _, _, _, err := db.getTxData(txSha)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// BUG(drahn) If there was an error beside non-existant deal with it.
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// FetchLocationBySha looks up the Tx sha information by name.
|
||||
func (db *LevelDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
err = fmt.Errorf("obsolete function")
|
||||
return
|
||||
}
|
||||
|
||||
// FetchTxUsedBySha returns the used/spent buffer for a given transaction.
|
||||
func (db *LevelDb) FetchTxUsedBySha(txSha *btcwire.ShaHash) (spentbuf []byte, err error) {
|
||||
db.dbLock.Lock()
|
||||
defer db.dbLock.Unlock()
|
||||
|
||||
_, _, _, spentbuf, err = db.getTxData(txSha)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return // spentbuf has the value already
|
||||
}
|
||||
|
||||
func (db *LevelDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) error {
|
||||
// delete me
|
||||
return fmt.Errorf("Deleted function")
|
||||
}
|
||||
Loading…
Reference in a new issue