diff --git a/ldb/block.go b/ldb/block.go new file mode 100644 index 00000000..8d211e75 --- /dev/null +++ b/ldb/block.go @@ -0,0 +1,262 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +// InsertBlockData stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +func (db *LevelDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertBlockData(sha, prevSha, buf) +} + +func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { + var blkHeight int64 + + key := sha.Bytes() + + data, err := db.bShaDb.Get(key, db.ro) + + if err != nil { + return 0, err + } + + // deserialize + dr := bytes.NewBuffer(data) + err = binary.Read(dr, binary.LittleEndian, &blkHeight) + if err != nil { + fmt.Printf("get getBlkLoc len %v\n", len(data)) + err = fmt.Errorf("Db Corrupt 0") + return 0, err + } + return blkHeight, nil +} + +func (db *LevelDb) getBlkByHeight(blkHeight int64) (rsha *btcwire.ShaHash, rbuf []byte, err error) { + var blkVal []byte + + key := int64ToKey(blkHeight) + + blkVal, err = db.bBlkDb.Get(key, db.ro) + if err != nil { + return // exists ??? + } + + var sha btcwire.ShaHash + + sha.SetBytes(blkVal[0:32]) + + blockdata := make([]byte, len(blkVal[32:])) + copy(blockdata[:], blkVal[32:]) + + return &sha, blockdata, nil +} + +func (db *LevelDb) getBlk(sha *btcwire.ShaHash) (rblkHeight int64, rbuf []byte, err error) { + var blkHeight int64 + + blkHeight, err = db.getBlkLoc(sha) + if err != nil { + return + } + + var buf []byte + + _, buf, err = db.getBlkByHeight(blkHeight) + if err != nil { + return + } + return blkHeight, buf, nil +} + +func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) error { + + // serialize + var lw bytes.Buffer + err := binary.Write(&lw, binary.LittleEndian, blkHeight) + if err != nil { + err = fmt.Errorf("Write Fail") + return err + } + shaKey := sha.Bytes() + + blkKey := int64ToKey(blkHeight) + + shaB := sha.Bytes() + blkVal := make([]byte, len(shaB)+len(buf)) + copy(blkVal[0:], shaB) + copy(blkVal[len(shaB):], buf) + + db.bShaBatch().Put(shaKey, lw.Bytes()) + + db.bBlkBatch().Put(blkKey, blkVal) + + return nil +} + +// insertSha stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +// insertSha shall be called with db lock held +func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, buf []byte) (blockid int64, err error) { + + oBlkHeight, err := db.getBlkLoc(prevSha) + + if err != nil { + // check current block count + // if count != 0 { + // err = btcdb.PrevShaMissing + // return + // } + oBlkHeight = -1 + } + + // TODO(drahn) check curfile filesize, increment curfile if this puts it over + blkHeight := oBlkHeight + 1 + + err = db.setBlk(sha, blkHeight, buf) + + if err != nil { + return + } + + // update the last block cache + db.lastBlkShaCached = true + db.lastBlkSha = *sha + db.lastBlkIdx = blkHeight + db.nextBlock = blkHeight + 1 + + return blkHeight, nil +} + +// fetchSha returns the datablock and pver for the given ShaHash. +func (db *LevelDb) fetchSha(sha *btcwire.ShaHash) (rbuf []byte, + rblkHeight int64, err error) { + var blkHeight int64 + var buf []byte + + blkHeight, buf, err = db.getBlk(sha) + if err != nil { + return + } + + return buf, blkHeight, nil +} + +// ExistsSha looks up the given block hash +// returns true if it is present in the database. +func (db *LevelDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // not in cache, try database + exists = db.blkExistsSha(sha) + return +} + +// blkExistsSha looks up the given block hash +// returns true if it is present in the database. +// CALLED WITH LOCK HELD +func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) bool { + + _, err := db.getBlkLoc(sha) + + if err != nil { + /* + should this warn if the failure is something besides does not exist ? + log.Warnf("blkExistsSha: fail %v", err) + */ + return false + } + return true +} + +// FetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *LevelDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.fetchBlockShaByHeight(height) +} + +// fetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *LevelDb) fetchBlockShaByHeight(height int64) (rsha *btcwire.ShaHash, err error) { + var sha *btcwire.ShaHash + sha, _, err = db.getBlkByHeight(height) + if err != nil { + return + } + + return sha, nil +} + +// FetchHeightRange looks up a range of blocks by the start and ending +// heights. Fetch is inclusive of the start height and exclusive of the +// ending height. To fetch all hashes from the start height until no +// more are present, use the special id `AllShas'. +func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + var endidx int64 + if endHeight == btcdb.AllShas { + endidx = startHeight + 500 + } else { + endidx = endHeight + } + + var shalist []btcwire.ShaHash + for height := startHeight; height < endidx; height++ { + // TODO(drahn) fix blkFile from height + + key := int64ToKey(height) + blkVal, lerr := db.bBlkDb.Get(key, db.ro) + if lerr != nil { + break + } + + var sha btcwire.ShaHash + sha.SetBytes(blkVal[0:32]) + shalist = append(shalist, sha) + } + + if err != nil { + return + } + //log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startHeight, endHeight, len(shalist), err) + + return shalist, nil +} + +// NewestSha returns the hash and block height of the most recent (end) block of +// the block chain. It will return the zero hash, -1 for the block height, and +// no error (nil) if there are not any blocks in the database yet. +func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + if db.lastBlkIdx == -1 { + err = fmt.Errorf("Empty Database") + return + } + sha := db.lastBlkSha + + return &sha, db.lastBlkIdx, nil +} + +func (db *LevelDb) NewIterateBlocks() (rbogus btcdb.BlockIterator, err error) { + err = fmt.Errorf("Not implemented") + return +} diff --git a/ldb/dbcache.go b/ldb/dbcache.go new file mode 100644 index 00000000..c5f74c90 --- /dev/null +++ b/ldb/dbcache.go @@ -0,0 +1,136 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" +) + +// FetchBlockBySha - return a btcutil Block, object may be a cached. +func (db *LevelDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + return db.fetchBlockBySha(sha) +} + +// fetchBlockBySha - return a btcutil Block, object may be a cached. +// Must be called with db lock held. +func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + + buf, height, err := db.fetchSha(sha) + if err != nil { + return + } + + blk, err = btcutil.NewBlockFromBytes(buf) + if err != nil { + return + } + blk.SetHeight(height) + + return +} + +// FetchTxByShaList given a array of ShaHash, look up the transactions +// and return them in a TxListReply array. +func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + replies := make([]*btcdb.TxListReply, len(txShaList)) + for i, txsha := range txShaList { + tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) + btxspent := []bool{} + if err == nil { + btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + } + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} + replies[i] = &txlre + } + return replies +} + +// fetchTxDataBySha returns several pieces of data regarding the given sha. +func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { + var pver uint32 + var blksha *btcwire.ShaHash + var blkHeight int64 + var txspent []byte + var txOff, txLen int + var blkbuf []byte + + blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha) + if err != nil { + err = btcdb.TxShaMissing + return + } + + blksha, blkbuf, err = db.getBlkByHeight(blkHeight) + if err != nil { + fmt.Printf("failed to get block %v %v\n", blkHeight, err) + return + } + + //log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n", + // txsha, blksha, blkHeight, txOff, txLen) + + txbuf := make([]byte, txLen) + copy(txbuf[:], blkbuf[txOff:txOff+txLen]) + rbuf := bytes.NewBuffer(txbuf) + + var tx btcwire.MsgTx + err = tx.Deserialize(rbuf) + if err != nil { + log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", + blkHeight, blksha, txOff, txLen) + return + } + + return &tx, txbuf, pver, blksha, blkHeight, txspent, nil +} + +// FetchTxAllBySha returns several pieces of data regarding the given sha. +func (db *LevelDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + tx, txbuf, pver, blksha, _, _, err := db.fetchTxDataBySha(txsha) + + return tx, txbuf, pver, blksha, nil +} + +// FetchTxBySha returns some data for the given Tx Sha. +func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { + rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha) + return +} + +// FetchTxBufBySha return the bytestream data and associated protocol version. +// for the given Tx Sha +func (db *LevelDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) { + _, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha) + return +} + +// InvalidateTxCache clear/release all cached transactions. +func (db *LevelDb) InvalidateTxCache() { +} + +// InvalidateTxCache clear/release all cached blocks. +func (db *LevelDb) InvalidateBlockCache() { +} + +// InvalidateCache clear/release all cached blocks and transactions. +func (db *LevelDb) InvalidateCache() { +} diff --git a/ldb/dbtest/dbtst.go b/ldb/dbtest/dbtst.go new file mode 100644 index 00000000..15d905bb --- /dev/null +++ b/ldb/dbtest/dbtst.go @@ -0,0 +1,56 @@ +// +package main + +import ( + "fmt" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +type tst struct { + key int + value string +} + +var dataset = []tst{ + //var dataset = []struct { key int, value string } { + {1, "one"}, + {2, "two"}, + {3, "three"}, + {4, "four"}, + {5, "five"}, +} + +func main() { + + ro := &opt.ReadOptions{} + wo := &opt.WriteOptions{} + + ldb, err := leveldb.OpenFile("dbfile", &opt.Options{Flag: opt.OFCreateIfMissing}) + if err != nil { + fmt.Printf("db open failed %v\n", err) + return + } + + batch := new(leveldb.Batch) + for _, datum := range dataset { + key := fmt.Sprintf("%v", datum.key) + batch.Put([]byte(key), []byte(datum.value)) + } + err = ldb.Write(batch, wo) + + for _, datum := range dataset { + key := fmt.Sprintf("%v", datum.key) + data, err := ldb.Get([]byte(key), ro) + + if err != nil { + fmt.Printf("db read failed %v\n", err) + } + + if string(data) != datum.value { + fmt.Printf("mismatched data from db key %v val %v db %v", key, datum.value, data) + } + } + fmt.Printf("completed\n") + ldb.Close() +} diff --git a/ldb/doc.go b/ldb/doc.go new file mode 100644 index 00000000..65906a93 --- /dev/null +++ b/ldb/doc.go @@ -0,0 +1,15 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package sqlite3 implements a sqlite3 instance of btcdb. + +sqlite provides a zero setup, single file database. It requires cgo +and the presence of the sqlite library and headers, but nothing else. +The performance is generally high although it goes down with database +size. + +Many of the block or tx specific functions for btcdb are in this subpackage. +*/ +package ldb diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go new file mode 100644 index 00000000..c6b6cafa --- /dev/null +++ b/ldb/insertremove_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/ldb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "os" + "path/filepath" + "testing" +) + +var tstBlocks []*btcutil.Block + +func loadblocks(t *testing.T) []*btcutil.Block { + if len(tstBlocks) != 0 { + return tstBlocks + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data: %v", err) + return nil + } + tstBlocks = blocks + return blocks +} + +func TestUnspentInsert(t *testing.T) { + testUnspentInsert(t, dbTmDefault) + testUnspentInsert(t, dbTmNormal) + testUnspentInsert(t, dbTmFast) +} + +// insert every block in the test chain +// after each insert, fetch all the tx affected by the latest +// block and verify that the the tx is spent/unspent +// new tx should be fully unspent, referenced tx should have +// the associated txout set to spent. +func testUnspentInsert(t *testing.T, mode int) { + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbuspnt1" + _ = os.RemoveAll(dbname) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + + case dbTmNoVerify: // validated block + t.Errorf("UnspentInsert test is not valid in NoVerify mode") + } + + // Since we are dealing with small dataset, reduce cache size + + blocks := loadblocks(t) +endtest: + for height := int64(0); height < int64(len(blocks)); height++ { + + block := blocks[height] + // look up inputs to this x + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + var txlookupList []*btcwire.ShaHash + var txOutList []*btcwire.ShaHash + var txInList []*btcwire.OutPoint + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + + txInList = append(txInList, &txin.PreviousOutpoint) + txneededList = append(txneededList, origintxsha) + txlookupList = append(txlookupList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + } + txshaname, _ := tx.TxSha() + txlookupList = append(txlookupList, &txshaname) + txOutList = append(txOutList, &txshaname) + } + + txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txneededmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txneededmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index) + } + } + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break endtest + } + + txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + for _, txo := range txOutList { + itxe := txlookupmap[*txo] + for i, spent := range itxe.TxSpent { + if spent == true { + t.Errorf("freshly inserted tx %v already spent %v", txo, i) + } + } + + } + if len(txInList) == 0 { + continue + } + dropblock := blocks[height-1] + dropsha, _ := dropblock.Sha() + + err = db.DropAfterBlockBySha(dropsha) + if err != nil { + t.Errorf("failed to drop block %v err %v", height, err) + break endtest + } + + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + if _, ok := txneededmap[*txe.Sha]; ok { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + newheight, err = db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + } +} diff --git a/ldb/internal_test.go b/ldb/internal_test.go new file mode 100644 index 00000000..3d6cabf3 --- /dev/null +++ b/ldb/internal_test.go @@ -0,0 +1,24 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +// FetchSha returns the datablock and pver for the given ShaHash. +// This is a testing only interface. +func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, + blkid int64, err error) { + sqldb, ok := db.(*LevelDb) + if !ok { + err = fmt.Errorf("Invalid data type") + return + } + buf, blkid, err = sqldb.fetchSha(sha) + return +} diff --git a/ldb/leveldb.go b/ldb/leveldb.go new file mode 100644 index 00000000..6605b368 --- /dev/null +++ b/ldb/leveldb.go @@ -0,0 +1,528 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + "os" + "path/filepath" + "runtime" + "sync" +) + +const ( + dbVersion int = 2 + dbMaxTransCnt = 20000 + dbMaxTransMem = 64 * 1024 * 1024 // 64 MB +) + +var log seelog.LoggerInterface = seelog.Disabled + +type tBlockInsertData struct { + sha btcwire.ShaHash + pver uint32 + buf []byte +} +type tTxInsertData struct { + txsha *btcwire.ShaHash + blockid int64 + txoff int + txlen int + usedbuf []byte +} + +type LevelDb struct { + // lock preventing multiple entry + dbLock sync.Mutex + + // leveldb pieces + bShaDb *leveldb.DB + bBlkDb *leveldb.DB + tShaDb *leveldb.DB + ro *opt.ReadOptions + wo *opt.WriteOptions + + bShabatch *leveldb.Batch + bBlkbatch *leveldb.Batch + + nextBlock int64 + + lastBlkShaCached bool + lastBlkSha btcwire.ShaHash + lastBlkIdx int64 + + txUpdateMap map[btcwire.ShaHash]*txUpdateObj +} + +var self = btcdb.DriverDB{DbType: "leveldb", Create: CreateDB, Open: OpenDB} + +func init() { + btcdb.AddDBDriver(self) +} + +// OpenDB opens an existing database for use. +func OpenDB(dbpath string) (btcdb.Db, error) { + log = btcdb.GetLog() + + db, err := openDB(dbpath, 0) + if err != nil { + return nil, err + } + log.Info("Opening DB\n") + + // Need to find last block and tx + + var lastknownblock, nextunknownblock, testblock int64 + + increment := int64(100000) + ldb := db.(*LevelDb) + + var lastSha *btcwire.ShaHash + // forward scan +blockforward: + for { + + sha, _, err := ldb.getBlkByHeight(testblock) + if err == nil { + // block is found + lastSha = sha + lastknownblock = testblock + testblock += increment + } else { + if testblock == 0 { + //no blocks in db, odd but ok. + return db, nil + } + nextunknownblock = testblock + break blockforward + } + } + + // narrow search +blocknarrow: + for { + testblock = (lastknownblock + nextunknownblock) / 2 + sha, _, err := ldb.getBlkByHeight(testblock) + if err == nil { + lastknownblock = testblock + lastSha = sha + } else { + nextunknownblock = testblock + } + if lastknownblock+1 == nextunknownblock { + break blocknarrow + } + } + + ldb.lastBlkSha = *lastSha + ldb.lastBlkIdx = lastknownblock + ldb.nextBlock = lastknownblock + 1 + + log.Info("Opening DB: completed\n") + + return db, nil +} + +func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { + var db LevelDb + var tbShaDb, ttShaDb, tbBlkDb *leveldb.DB + defer func() { + if err == nil { + db.bShaDb = tbShaDb + db.bBlkDb = tbBlkDb + db.tShaDb = ttShaDb + + db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} + + pbdb = &db + } + }() + + if flag&opt.OFCreateIfMissing == opt.OFCreateIfMissing { + err = os.Mkdir(dbpath, 0750) + if err != nil { + log.Errorf("mkdir failed %v %v", dbpath, err) + return + } + } else { + _, err = os.Stat(dbpath) + if err != nil { + err = btcdb.DbDoesNotExist + return + } + } + + bShaName := filepath.Join(dbpath, "bSha.ldb") + tbShaDb, err = leveldb.OpenFile(bShaName, &opt.Options{Flag: flag}) + if err != nil { + return + } + bBlkName := filepath.Join(dbpath, "bBlk.ldb") + tbBlkDb, err = leveldb.OpenFile(bBlkName, &opt.Options{Flag: flag}) + if err != nil { + return + } + tShaName := filepath.Join(dbpath, "tSha.ldb") + ttShaDb, err = leveldb.OpenFile(tShaName, &opt.Options{Flag: flag}) + if err != nil { + return + } + + return +} + +// CreateDB creates, initializes and opens a database for use. +func CreateDB(dbpath string) (btcdb.Db, error) { + log = btcdb.GetLog() + + // No special setup needed, just OpenBB + return openDB(dbpath, opt.OFCreateIfMissing) +} + +func (db *LevelDb) close() { + db.bShaDb.Close() + db.bBlkDb.Close() + db.tShaDb.Close() +} + +// Sync verifies that the database is coherent on disk, +// and no outstanding transactions are in flight. +func (db *LevelDb) Sync() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // while specified by the API, does nothing + // however does grab lock to verify it does not return until other operations are complete. +} + +// Close cleanly shuts down database, syncing all data. +func (db *LevelDb) Close() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + db.close() +} + +// DropAfterBlockBySha will remove any blocks from the database after +// the given block. +func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { + db.dbLock.Lock() + defer db.dbLock.Unlock() + defer db.processBatches() + + startheight := db.nextBlock - 1 + + keepidx, err := db.getBlkLoc(sha) + if err != nil { + // should the error here be normalized ? + log.Infof("block loc failed %v ", sha) + return err + } + + for height := startheight; height > keepidx; height = height - 1 { + var blk *btcutil.Block + blksha, buf, err := db.getBlkByHeight(height) + if err != nil { + return err + } + blk, err = btcutil.NewBlockFromBytes(buf) + if err != nil { + return err + } + + for _, tx := range blk.MsgBlock().Transactions { + err = db.unSpend(tx) + if err != nil { + return err + } + } + // rather than iterate the list of tx backward, do it twice. + for _, tx := range blk.MsgBlock().Transactions { + txSha, _ := tx.TxSha() + var txUo txUpdateObj + txUo.delete = true + db.txUpdateMap[txSha] = &txUo + } + db.bShaBatch().Delete(shaToKey(blksha)) + db.bBlkBatch().Delete(int64ToKey(height)) + } + + db.nextBlock = keepidx + 1 + + return nil +} + +// InsertBlock inserts raw block and transaction data from a block into the +// database. The first block inserted into the database will be treated as the +// genesis block. Every subsequent block insert requires the referenced parent +// block to already exist. +func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + defer db.processBatches() + + blocksha, err := block.Sha() + if err != nil { + log.Warnf("Failed to compute block sha %v", blocksha) + return + } + mblock := block.MsgBlock() + rawMsg, err := block.Bytes() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + txloc, err := block.TxLoc() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + + // Insert block into database + newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, + rawMsg) + if err != nil { + log.Warnf("Failed to insert block %v %v %v", blocksha, + &mblock.Header.PrevBlock, err) + return + } + + // At least two blocks in the long past were generated by faulty + // miners, the sha of the transaction exists in a previous block, + // detect this condition and 'accept' the block. + for txidx, tx := range mblock.Transactions { + var txsha btcwire.ShaHash + txsha, err = tx.TxSha() + if err != nil { + log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) + return + } + // Some old blocks contain duplicate transactions + // Attempt to cleanly bypass this problem + // http://blockexplorer.com/b/91842 + // http://blockexplorer.com/b/91880 + if newheight == 91842 { + dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + //log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + if newheight == 91880 { + dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + //log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + spentbuflen := (len(tx.TxOut) + 7) / 8 + spentbuf := make([]byte, spentbuflen, spentbuflen) + if len(tx.TxOut)%8 != 0 { + for i := uint(len(tx.TxOut) % 8); i < 8; i++ { + spentbuf[spentbuflen-1] |= (byte(1) << i) + } + } + + err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) + if err != nil { + log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + + return + } + err = db.doSpend(tx) + if err != nil { + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + + return + } + } + return newheight, nil +} + +// SetDBInsertMode provides hints to the database to how the application +// is running this allows the database to work in optimized modes when the +// database may be very busy. +func (db *LevelDb) SetDBInsertMode(newmode btcdb.InsertMode) { + + // special modes are not supported +} + +// doSpend iterates all TxIn in a bitcoin transaction marking each associated +// TxOut as spent. +func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + //log.Infof("spending %v %v", &inTxSha, inTxidx) + + err := db.setSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +// unSpend iterates all TxIn in a bitcoin transaction marking each associated +// TxOut as unspent. +func (db *LevelDb) unSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + err := db.clearSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +func (db *LevelDb) setSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, true) +} + +func (db *LevelDb) clearSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, false) +} + +func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bool) error { + var txUo *txUpdateObj + var ok bool + + if txUo, ok = db.txUpdateMap[*txsha]; !ok { + // not cached, load from db + var txU txUpdateObj + blkHeight, txOff, txLen, spentData, err := db.getTxData(txsha) + if err != nil { + return err + } + + txU.txSha = txsha + txU.blkHeight = blkHeight + txU.txoff = txOff + txU.txlen = txLen + txU.spentData = spentData + + txUo = &txU + } + + byteidx := idx / 8 + byteoff := idx % 8 + + if set { + txUo.spentData[byteidx] |= (byte(1) << byteoff) + } else { + txUo.spentData[byteidx] &= ^(byte(1) << byteoff) + } + + db.txUpdateMap[*txsha] = txUo + + return nil +} + +func intToKey(keyint int) []byte { + key := fmt.Sprintf("%d", keyint) + return []byte(key) +} +func int64ToKey(keyint int64) []byte { + key := fmt.Sprintf("%d", keyint) + return []byte(key) +} + +func shaToKey(sha *btcwire.ShaHash) []byte { + return sha.Bytes() +} + +func (db *LevelDb) bShaBatch() *leveldb.Batch { + if db.bShabatch == nil { + db.bShabatch = new(leveldb.Batch) + } + return db.bShabatch +} + +func (db *LevelDb) bBlkBatch() *leveldb.Batch { + if db.bBlkbatch == nil { + db.bBlkbatch = new(leveldb.Batch) + } + return db.bBlkbatch +} + +func (db *LevelDb) processBatches() { + var err error + if db.bShabatch != nil { + err = db.bShaDb.Write(db.bShabatch, db.wo) + db.bShabatch.Reset() + db.bShabatch = nil + if err != nil { + return + } + } + if db.bBlkbatch != nil { + err = db.bBlkDb.Write(db.bBlkbatch, db.wo) + db.bBlkbatch.Reset() + db.bBlkbatch = nil + if err != nil { + return + } + } + + if len(db.txUpdateMap) != 0 { + tShabatch := new(leveldb.Batch) + + for txSha, txU := range db.txUpdateMap { + key := shaToKey(&txSha) + if txU.delete { + //log.Infof("deleting tx %v", txSha) + tShabatch.Delete(key) + } else { + //log.Infof("inserting tx %v", txSha) + txdat, err := db.formatTx(txU) + if err != nil { + return + } + tShabatch.Put(key, txdat) + } + } + + err = db.tShaDb.Write(tShabatch, db.wo) + tShabatch.Reset() + if err != nil { + return + } + db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} + runtime.GC() + } + +} + +func (db *LevelDb) RollbackClose() { + db.close() +} diff --git a/ldb/operational_test.go b/ldb/operational_test.go new file mode 100644 index 00000000..1000bd66 --- /dev/null +++ b/ldb/operational_test.go @@ -0,0 +1,394 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "compress/bzip2" + "encoding/binary" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +var network = btcwire.MainNet + +const ( + dbTmDefault = iota + dbTmNormal + dbTmFast + dbTmNoVerify +) + +func TestOperational(t *testing.T) { + testOperationalMode(t, dbTmDefault) + testOperationalMode(t, dbTmNormal) + testOperationalMode(t, dbTmFast) + testOperationalMode(t, dbTmNoVerify) +} + +func testOperationalMode(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop1" + _ = os.RemoveAll(dbname) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertValidatedInput) + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data for mode %v: %v", + mode, err) + return + } + + err = nil +out: + for height := int64(0); height < int64(len(blocks)); height++ { + block := blocks[height] + if mode != dbTmNoVerify { + // except for NoVerify which does not allow lookups check inputs + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + txneededList = append(txneededList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + _, _, _, _, err := db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, _, err = db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, err = db.FetchTxBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, err = db.FetchTxBufBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, err = db.FetchTxUsedBySha(origintxsha) + if err != nil { + t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) + } + } + } + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break out + } + } + + } + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break out + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break out + } + + newSha, blkid, err := db.NewestSha() + if err != nil { + t.Errorf("failed to obtain latest sha %v %v", height, err) + } + + if blkid != height { + t.Errorf("height doe not match latest block height %v %v", blkid, height, err) + } + + blkSha, _ := block.Sha() + if *newSha != *blkSha { + t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err) + } + } + + // now that db is populated, do some additional test + testFetchRangeHeight(t, db, blocks) + + switch mode { + case dbTmDefault: // default + // no cleanup + case dbTmNormal: // explicit normal + // no cleanup + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertNormal) + } +} + +func TestBackout(t *testing.T) { + testBackout(t, dbTmDefault) + testBackout(t, dbTmNormal) + testBackout(t, dbTmFast) +} + +func testBackout(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop2" + _ = os.RemoveAll(dbname) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if len(blocks) < 120 { + t.Errorf("test data too small") + return + } + + err = nil + for height := int64(0); height < int64(len(blocks)); height++ { + if height == 100 { + t.Logf("Syncing at block height 100") + db.Sync() + } + if height == 120 { + t.Logf("Simulating unexpected application quit") + // Simulate unexpected application quit + db.RollbackClose() + break + } + + block := blocks[height] + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break + } + } + + // db was closed at height 120, so no cleanup is possible. + + // reopen db + db, err = btcdb.OpenDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer db.Close() + + sha, err := blocks[99].Sha() + if err != nil { + t.Errorf("failed to get block 99 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err != nil { + t.Errorf("failed to load block 99 from db %v", err) + } + + sha, err = blocks[119].Sha() + if err != nil { + t.Errorf("failed to get block 110 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err != nil { + t.Errorf("loaded block 119 from db") + return + } + + block := blocks[119] + mblock := block.MsgBlock() + txsha, err := mblock.Transactions[0].TxSha() + exists := db.ExistsTxSha(&txsha) + if !exists { + t.Errorf("tx %v not located db\n", txsha) + } + + _, _, _, err = db.FetchTxBySha(&txsha) + if err != nil { + t.Errorf("tx %v not located db\n", txsha) + return + } + _, err = db.FetchTxUsedBySha(&txsha) + if err != nil { + t.Errorf("tx %v not located db\n", txsha) + return + } +} + +func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + var dr io.Reader + var fi io.ReadCloser + fi, err = os.Open(testdatafile) + if err != nil { + t.Errorf("failed to open file %v, err %v", testdatafile, err) + return + } + if strings.HasSuffix(testdatafile, ".bz2") { + z := bzip2.NewReader(fi) + dr = z + } else { + dr = fi + } + + defer func() { + if err := fi.Close(); err != nil { + t.Errorf("failed to close file %v %v", testdatafile, err) + } + }() + + // Set the first block as the genesis block. + genesis := btcutil.NewBlock(&btcwire.GenesisBlock) + blocks = append(blocks, genesis) + + var block *btcutil.Block + err = nil + for height := int64(1); err == nil; height++ { + var rintbuf uint32 + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + if err == io.EOF { + // hit end of file at expected offset: no warning + height-- + err = nil + break + } + if err != nil { + t.Errorf("failed to load network type, err %v", err) + break + } + if rintbuf != uint32(network) { + t.Errorf("Block doesn't match network: %v expects %v", + rintbuf, network) + break + } + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + blocklen := rintbuf + + rbytes := make([]byte, blocklen) + + // read block + dr.Read(rbytes) + + block, err = btcutil.NewBlockFromBytes(rbytes) + if err != nil { + t.Errorf("failed to parse block %v", height) + return + } + blocks = append(blocks, block) + } + return +} + +func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { + + var testincrement int64 = 50 + var testcnt int64 = 100 + + shanames := make([]*btcwire.ShaHash, len(blocks)) + + nBlocks := int64(len(blocks)) + + for i := range blocks { + blockSha, err := blocks[i].Sha() + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err) + } + shanames[i] = blockSha + } + + for startheight := int64(0); startheight < nBlocks; startheight += testincrement { + endheight := startheight + testcnt + + if endheight > nBlocks { + endheight = btcdb.AllShas + } + + shalist, err := db.FetchHeightRange(startheight, endheight) + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err) + } + + if endheight == btcdb.AllShas { + if int64(len(shalist)) != nBlocks-startheight { + t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) + } + } else { + if int64(len(shalist)) != testcnt { + t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist)) + } + } + + for i := range shalist { + sha0 := *shanames[int64(i)+startheight] + sha1 := shalist[i] + if sha0 != sha1 { + t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v: %v %v ", int64(i)+startheight, startheight, endheight, sha0, sha1) + } + } + } + +} diff --git a/ldb/testdata/blocks1-256.bz2 b/ldb/testdata/blocks1-256.bz2 new file mode 100644 index 00000000..6b8bda44 Binary files /dev/null and b/ldb/testdata/blocks1-256.bz2 differ diff --git a/ldb/tx.go b/ldb/tx.go new file mode 100644 index 00000000..300f599f --- /dev/null +++ b/ldb/tx.go @@ -0,0 +1,175 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "encoding/binary" + "fmt" + //"github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +type txUpdateObj struct { + txSha *btcwire.ShaHash + blkHeight int64 + txoff int + txlen int + spentData []byte + delete bool +} + +// InsertTx inserts a tx hash and its associated data into the database. +func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertTx(txsha, height, txoff, txlen, spentbuf) +} + +// insertTx inserts a tx hash and its associated data into the database. +// Must be called with db lock held. +func (db *LevelDb) insertTx(txSha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) { + var txU txUpdateObj + + txU.txSha = txSha + txU.blkHeight = height + txU.txoff = txoff + txU.txlen = txlen + txU.spentData = spentbuf + + db.txUpdateMap[*txSha] = &txU + + return nil +} + +// formatTx generates the value buffer for the Tx db. +func (db *LevelDb) formatTx(txu *txUpdateObj) ([]byte, error) { + + blkHeight := txu.blkHeight + txoff := txu.txoff + txlen := txu.txlen + spentbuf := txu.spentData + + txOff := int32(txoff) + txLen := int32(txlen) + + var txW bytes.Buffer + + err := binary.Write(&txW, binary.LittleEndian, blkHeight) + if err != nil { + fmt.Printf("fail encoding blkHeight %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, txOff) + if err != nil { + fmt.Printf("fail encoding txoff %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, txLen) + if err != nil { + fmt.Printf("fail encoding txlen %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, spentbuf) + if err != nil { + fmt.Printf("fail encoding spentbuf %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + return txW.Bytes(), nil +} + +func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (rblkHeight int64, + rtxOff int, rtxLen int, rspentBuf []byte, err error) { + var buf []byte + + key := shaToKey(txsha) + buf, err = db.tShaDb.Get(key, db.ro) + if err != nil { + return + } + + var blkHeight int64 + var txOff, txLen int32 + dr := bytes.NewBuffer(buf) + err = binary.Read(dr, binary.LittleEndian, &blkHeight) + if err != nil { + err = fmt.Errorf("Db Corrupt 1") + return + } + err = binary.Read(dr, binary.LittleEndian, &txOff) + if err != nil { + err = fmt.Errorf("Db Corrupt 2") + return + } + err = binary.Read(dr, binary.LittleEndian, &txLen) + if err != nil { + err = fmt.Errorf("Db Corrupt 3") + return + } + // remainder of buffer is spentbuf + spentBuf := make([]byte, dr.Len()) + err = binary.Read(dr, binary.LittleEndian, spentBuf) + if err != nil { + fmt.Printf("fail encoding spentbuf %v\n", err) + err = fmt.Errorf("Db Corrupt 4") + return + } + return blkHeight, int(txOff), int(txLen), spentBuf, nil +} + +// ExistsTxSha returns if the given tx sha exists in the database +func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.existsTxSha(txsha) +} + +// existsTxSha returns if the given tx sha exists in the database.o +// Must be called with the db lock held. +func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) { + _, _, _, _, err := db.getTxData(txSha) + if err == nil { + return true + } + + // BUG(drahn) If there was an error beside non-existant deal with it. + + return false +} + +// FetchLocationBySha looks up the Tx sha information by name. +func (db *LevelDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + err = fmt.Errorf("obsolete function") + return +} + +// FetchTxUsedBySha returns the used/spent buffer for a given transaction. +func (db *LevelDb) FetchTxUsedBySha(txSha *btcwire.ShaHash) (spentbuf []byte, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + _, _, _, spentbuf, err = db.getTxData(txSha) + if err != nil { + return + } + return // spentbuf has the value already +} + +func (db *LevelDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) error { + // delete me + return fmt.Errorf("Deleted function") +}